2024-11-28 03:40:43,279 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-28 03:40:43,296 main DEBUG Took 0.013524 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 03:40:43,296 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 03:40:43,297 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 03:40:43,298 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 03:40:43,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,308 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 03:40:43,337 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,350 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,351 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,351 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,352 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,352 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,357 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,357 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,358 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,358 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,359 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,360 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,361 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,361 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,362 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,362 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,363 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,364 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,365 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:40:43,366 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,366 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 03:40:43,373 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:40:43,384 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 03:40:43,390 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 03:40:43,391 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 03:40:43,401 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 03:40:43,402 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 03:40:43,429 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 03:40:43,432 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 03:40:43,440 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 03:40:43,441 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 03:40:43,443 main DEBUG createAppenders(={Console}) 2024-11-28 03:40:43,446 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-28 03:40:43,446 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-28 03:40:43,447 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-28 03:40:43,448 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 03:40:43,448 main DEBUG OutputStream closed 2024-11-28 03:40:43,449 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 03:40:43,449 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 03:40:43,449 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-28 03:40:43,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 03:40:43,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 03:40:43,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 03:40:43,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 03:40:43,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 03:40:43,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 03:40:43,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 03:40:43,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 03:40:43,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 03:40:43,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 03:40:43,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 03:40:43,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 03:40:43,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 03:40:43,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 03:40:43,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 03:40:43,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 03:40:43,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 03:40:43,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 03:40:43,573 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 03:40:43,573 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-28 03:40:43,573 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 03:40:43,574 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-28T03:40:44,001 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64 2024-11-28 03:40:44,007 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 03:40:44,007 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T03:40:44,022 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-28T03:40:44,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-28T03:40:44,086 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/cluster_bc0417c7-3664-f9fe-1add-573adc96de9e, deleteOnExit=true 2024-11-28T03:40:44,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-28T03:40:44,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/test.cache.data in system properties and HBase conf 2024-11-28T03:40:44,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T03:40:44,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/hadoop.log.dir in system properties and HBase conf 2024-11-28T03:40:44,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T03:40:44,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T03:40:44,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T03:40:44,270 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T03:40:44,403 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T03:40:44,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T03:40:44,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T03:40:44,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T03:40:44,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T03:40:44,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T03:40:44,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T03:40:44,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T03:40:44,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T03:40:44,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T03:40:44,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/nfs.dump.dir in system properties and HBase conf 2024-11-28T03:40:44,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/java.io.tmpdir in system properties and HBase conf 2024-11-28T03:40:44,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T03:40:44,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T03:40:44,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T03:40:45,434 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-28T03:40:45,533 INFO [Time-limited test {}] log.Log(170): Logging initialized @3271ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T03:40:45,645 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T03:40:45,720 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T03:40:45,759 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T03:40:45,759 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T03:40:45,761 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T03:40:45,779 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T03:40:45,782 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f9fd429{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/hadoop.log.dir/,AVAILABLE} 2024-11-28T03:40:45,783 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40029c9a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T03:40:46,049 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a5ffc6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/java.io.tmpdir/jetty-localhost-33267-hadoop-hdfs-3_4_1-tests_jar-_-any-13150562804301654348/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T03:40:46,065 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f30a77e{HTTP/1.1, (http/1.1)}{localhost:33267} 2024-11-28T03:40:46,065 INFO [Time-limited test {}] server.Server(415): Started @3805ms 2024-11-28T03:40:46,527 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T03:40:46,536 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T03:40:46,539 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T03:40:46,539 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T03:40:46,539 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T03:40:46,540 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d058786{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/hadoop.log.dir/,AVAILABLE} 2024-11-28T03:40:46,541 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@50fd64ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T03:40:46,676 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58e0e813{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/java.io.tmpdir/jetty-localhost-33987-hadoop-hdfs-3_4_1-tests_jar-_-any-4176378105609817761/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T03:40:46,677 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@716e8981{HTTP/1.1, (http/1.1)}{localhost:33987} 2024-11-28T03:40:46,678 INFO [Time-limited test {}] server.Server(415): Started @4417ms 2024-11-28T03:40:46,743 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T03:40:47,436 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/cluster_bc0417c7-3664-f9fe-1add-573adc96de9e/data/data1/current/BP-1741671718-172.17.0.2-1732765245159/current, will proceed with Du for space computation calculation, 2024-11-28T03:40:47,436 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/cluster_bc0417c7-3664-f9fe-1add-573adc96de9e/data/data2/current/BP-1741671718-172.17.0.2-1732765245159/current, will proceed with Du for space computation calculation, 2024-11-28T03:40:47,513 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T03:40:47,591 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x595bcf8519779906 with lease ID 0x339168b8c265daa7: Processing first storage report for DS-90071d3e-5aaf-4155-97e6-98eee4eb70d9 from datanode DatanodeRegistration(127.0.0.1:35257, datanodeUuid=1af0cbc8-7da9-4027-a904-1e366931edcb, infoPort=39407, infoSecurePort=0, ipcPort=43603, storageInfo=lv=-57;cid=testClusterID;nsid=582012311;c=1732765245159) 2024-11-28T03:40:47,593 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x595bcf8519779906 with lease ID 0x339168b8c265daa7: from storage DS-90071d3e-5aaf-4155-97e6-98eee4eb70d9 node DatanodeRegistration(127.0.0.1:35257, datanodeUuid=1af0cbc8-7da9-4027-a904-1e366931edcb, infoPort=39407, infoSecurePort=0, ipcPort=43603, storageInfo=lv=-57;cid=testClusterID;nsid=582012311;c=1732765245159), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-28T03:40:47,594 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x595bcf8519779906 with lease ID 0x339168b8c265daa7: Processing first storage report for DS-e053883f-d100-4ceb-9bdd-2aced46d10b9 from datanode DatanodeRegistration(127.0.0.1:35257, datanodeUuid=1af0cbc8-7da9-4027-a904-1e366931edcb, infoPort=39407, infoSecurePort=0, ipcPort=43603, storageInfo=lv=-57;cid=testClusterID;nsid=582012311;c=1732765245159) 2024-11-28T03:40:47,594 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x595bcf8519779906 with lease ID 0x339168b8c265daa7: from storage DS-e053883f-d100-4ceb-9bdd-2aced46d10b9 node DatanodeRegistration(127.0.0.1:35257, datanodeUuid=1af0cbc8-7da9-4027-a904-1e366931edcb, infoPort=39407, infoSecurePort=0, ipcPort=43603, storageInfo=lv=-57;cid=testClusterID;nsid=582012311;c=1732765245159), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T03:40:47,607 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64 2024-11-28T03:40:47,738 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/cluster_bc0417c7-3664-f9fe-1add-573adc96de9e/zookeeper_0, clientPort=50439, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/cluster_bc0417c7-3664-f9fe-1add-573adc96de9e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/cluster_bc0417c7-3664-f9fe-1add-573adc96de9e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T03:40:47,748 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50439 2024-11-28T03:40:47,760 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:40:47,763 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:40:48,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741825_1001 (size=7) 2024-11-28T03:40:48,478 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72 with version=8 2024-11-28T03:40:48,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/hbase-staging 2024-11-28T03:40:48,575 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T03:40:48,838 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3a546af21488:0 server-side Connection retries=45 2024-11-28T03:40:48,850 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T03:40:48,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T03:40:48,855 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T03:40:48,856 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T03:40:48,856 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T03:40:49,011 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T03:40:49,085 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T03:40:49,097 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T03:40:49,102 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T03:40:49,135 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1559 (auto-detected) 2024-11-28T03:40:49,136 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T03:40:49,160 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38327 2024-11-28T03:40:49,195 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38327 connecting to ZooKeeper ensemble=127.0.0.1:50439 2024-11-28T03:40:49,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:383270x0, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T03:40:49,235 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38327-0x1003c3721c70000 connected 2024-11-28T03:40:49,272 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:40:49,274 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:40:49,287 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T03:40:49,292 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72, hbase.cluster.distributed=false 2024-11-28T03:40:49,322 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T03:40:49,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38327 2024-11-28T03:40:49,337 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38327 2024-11-28T03:40:49,338 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38327 2024-11-28T03:40:49,343 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38327 2024-11-28T03:40:49,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38327 2024-11-28T03:40:49,501 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3a546af21488:0 server-side Connection retries=45 2024-11-28T03:40:49,503 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T03:40:49,504 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T03:40:49,504 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T03:40:49,504 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T03:40:49,505 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T03:40:49,508 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T03:40:49,511 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T03:40:49,512 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34671 2024-11-28T03:40:49,514 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34671 connecting to ZooKeeper ensemble=127.0.0.1:50439 2024-11-28T03:40:49,515 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:40:49,518 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:40:49,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:346710x0, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T03:40:49,531 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T03:40:49,531 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34671-0x1003c3721c70001 connected 2024-11-28T03:40:49,538 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T03:40:49,558 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T03:40:49,562 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T03:40:49,567 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T03:40:49,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34671 2024-11-28T03:40:49,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34671 2024-11-28T03:40:49,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34671 2024-11-28T03:40:49,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34671 2024-11-28T03:40:49,584 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34671 2024-11-28T03:40:49,607 DEBUG [M:0;3a546af21488:38327 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3a546af21488:38327 2024-11-28T03:40:49,608 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3a546af21488,38327,1732765248628 2024-11-28T03:40:49,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T03:40:49,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T03:40:49,619 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3a546af21488,38327,1732765248628 2024-11-28T03:40:49,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T03:40:49,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:49,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:49,651 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T03:40:49,652 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3a546af21488,38327,1732765248628 from backup master directory 2024-11-28T03:40:49,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3a546af21488,38327,1732765248628 2024-11-28T03:40:49,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T03:40:49,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T03:40:49,656 WARN [master/3a546af21488:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T03:40:49,657 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3a546af21488,38327,1732765248628 2024-11-28T03:40:49,659 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T03:40:49,661 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T03:40:49,740 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/hbase.id] with ID: 68328222-4272-433b-8f44-b2792fc37682 2024-11-28T03:40:49,740 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/.tmp/hbase.id 2024-11-28T03:40:49,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741826_1002 (size=42) 2024-11-28T03:40:50,157 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/.tmp/hbase.id]:[hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/hbase.id] 2024-11-28T03:40:50,212 INFO [master/3a546af21488:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:40:50,217 INFO [master/3a546af21488:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T03:40:50,242 INFO [master/3a546af21488:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-28T03:40:50,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:50,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:50,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741827_1003 (size=196) 2024-11-28T03:40:50,687 INFO [master/3a546af21488:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:40:50,690 INFO [master/3a546af21488:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T03:40:50,724 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:50,731 INFO [master/3a546af21488:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T03:40:50,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741828_1004 (size=1189) 2024-11-28T03:40:51,205 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store 2024-11-28T03:40:51,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741829_1005 (size=34) 2024-11-28T03:40:51,634 INFO [master/3a546af21488:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T03:40:51,638 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:40:51,639 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T03:40:51,639 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:40:51,639 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:40:51,641 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T03:40:51,641 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:40:51,641 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:40:51,642 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732765251639Disabling compacts and flushes for region at 1732765251639Disabling writes for close at 1732765251641 (+2 ms)Writing region close event to WAL at 1732765251641Closed at 1732765251641 2024-11-28T03:40:51,645 WARN [master/3a546af21488:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/.initializing 2024-11-28T03:40:51,645 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/WALs/3a546af21488,38327,1732765248628 2024-11-28T03:40:51,654 INFO [master/3a546af21488:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T03:40:51,668 INFO [master/3a546af21488:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3a546af21488%2C38327%2C1732765248628, suffix=, logDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/WALs/3a546af21488,38327,1732765248628, archiveDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/oldWALs, maxLogs=10 2024-11-28T03:40:51,692 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/WALs/3a546af21488,38327,1732765248628/3a546af21488%2C38327%2C1732765248628.1732765251673, exclude list is [], retry=0 2024-11-28T03:40:51,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35257,DS-90071d3e-5aaf-4155-97e6-98eee4eb70d9,DISK] 2024-11-28T03:40:51,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-28T03:40:51,748 INFO [master/3a546af21488:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/WALs/3a546af21488,38327,1732765248628/3a546af21488%2C38327%2C1732765248628.1732765251673 2024-11-28T03:40:51,749 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39407:39407)] 2024-11-28T03:40:51,750 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:40:51,750 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:40:51,754 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,755 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,793 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T03:40:51,823 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:51,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:40:51,827 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T03:40:51,830 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:51,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:40:51,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T03:40:51,834 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:51,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:40:51,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,838 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T03:40:51,838 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:51,839 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:40:51,839 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,844 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,845 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,851 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,852 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,855 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T03:40:51,858 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:40:51,863 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:40:51,864 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75479403, jitterRate=0.12473075091838837}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T03:40:51,869 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732765251767Initializing all the Stores at 1732765251769 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732765251770 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765251770Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765251771 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765251771Cleaning up temporary data from old regions at 1732765251852 (+81 ms)Region opened successfully at 1732765251869 (+17 ms) 2024-11-28T03:40:51,871 INFO [master/3a546af21488:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T03:40:51,905 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcc19d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3a546af21488/172.17.0.2:0 2024-11-28T03:40:51,938 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T03:40:51,950 INFO [master/3a546af21488:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T03:40:51,950 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T03:40:51,954 INFO [master/3a546af21488:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T03:40:51,956 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T03:40:51,961 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-28T03:40:51,961 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T03:40:51,989 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T03:40:51,998 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T03:40:52,001 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T03:40:52,003 INFO [master/3a546af21488:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T03:40:52,004 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T03:40:52,006 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T03:40:52,008 INFO [master/3a546af21488:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T03:40:52,011 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T03:40:52,013 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T03:40:52,014 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T03:40:52,016 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T03:40:52,033 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T03:40:52,035 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T03:40:52,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T03:40:52,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T03:40:52,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:52,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:52,042 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3a546af21488,38327,1732765248628, sessionid=0x1003c3721c70000, setting cluster-up flag (Was=false) 2024-11-28T03:40:52,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:52,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:52,059 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T03:40:52,061 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3a546af21488,38327,1732765248628 2024-11-28T03:40:52,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:52,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:52,072 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T03:40:52,073 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3a546af21488,38327,1732765248628 2024-11-28T03:40:52,080 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T03:40:52,090 INFO [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(746): ClusterId : 68328222-4272-433b-8f44-b2792fc37682 2024-11-28T03:40:52,092 DEBUG [RS:0;3a546af21488:34671 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T03:40:52,097 DEBUG [RS:0;3a546af21488:34671 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T03:40:52,097 DEBUG [RS:0;3a546af21488:34671 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T03:40:52,100 DEBUG [RS:0;3a546af21488:34671 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T03:40:52,101 DEBUG [RS:0;3a546af21488:34671 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5abd959e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3a546af21488/172.17.0.2:0 2024-11-28T03:40:52,116 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3a546af21488:34671 2024-11-28T03:40:52,119 INFO [RS:0;3a546af21488:34671 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T03:40:52,119 INFO [RS:0;3a546af21488:34671 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T03:40:52,119 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T03:40:52,122 INFO [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(2659): reportForDuty to master=3a546af21488,38327,1732765248628 with port=34671, startcode=1732765249447 2024-11-28T03:40:52,137 DEBUG [RS:0;3a546af21488:34671 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T03:40:52,164 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T03:40:52,174 INFO [master/3a546af21488:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T03:40:52,178 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49879, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T03:40:52,184 INFO [master/3a546af21488:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T03:40:52,186 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38327 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:52,191 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3a546af21488,38327,1732765248628 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T03:40:52,199 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3a546af21488:0, corePoolSize=5, maxPoolSize=5 2024-11-28T03:40:52,199 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3a546af21488:0, corePoolSize=5, maxPoolSize=5 2024-11-28T03:40:52,200 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3a546af21488:0, corePoolSize=5, maxPoolSize=5 2024-11-28T03:40:52,200 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3a546af21488:0, corePoolSize=5, maxPoolSize=5 2024-11-28T03:40:52,200 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3a546af21488:0, corePoolSize=10, maxPoolSize=10 2024-11-28T03:40:52,201 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,201 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3a546af21488:0, corePoolSize=2, maxPoolSize=2 2024-11-28T03:40:52,201 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,204 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732765282204 2024-11-28T03:40:52,206 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T03:40:52,207 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T03:40:52,207 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T03:40:52,207 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T03:40:52,212 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T03:40:52,212 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T03:40:52,212 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T03:40:52,213 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T03:40:52,214 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:52,214 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T03:40:52,214 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,218 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T03:40:52,219 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T03:40:52,219 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-28T03:40:52,220 WARN [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-28T03:40:52,220 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T03:40:52,222 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T03:40:52,222 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T03:40:52,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741831_1007 (size=1321) 2024-11-28T03:40:52,224 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3a546af21488:0:becomeActiveMaster-HFileCleaner.large.0-1732765252224,5,FailOnTimeoutGroup] 2024-11-28T03:40:52,225 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3a546af21488:0:becomeActiveMaster-HFileCleaner.small.0-1732765252224,5,FailOnTimeoutGroup] 2024-11-28T03:40:52,226 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,226 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T03:40:52,227 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,227 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,321 INFO [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(2659): reportForDuty to master=3a546af21488,38327,1732765248628 with port=34671, startcode=1732765249447 2024-11-28T03:40:52,323 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38327 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3a546af21488,34671,1732765249447 2024-11-28T03:40:52,326 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38327 {}] master.ServerManager(517): Registering regionserver=3a546af21488,34671,1732765249447 2024-11-28T03:40:52,336 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72 2024-11-28T03:40:52,336 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38161 2024-11-28T03:40:52,336 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T03:40:52,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T03:40:52,341 DEBUG [RS:0;3a546af21488:34671 {}] zookeeper.ZKUtil(111): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3a546af21488,34671,1732765249447 2024-11-28T03:40:52,341 WARN [RS:0;3a546af21488:34671 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T03:40:52,341 INFO [RS:0;3a546af21488:34671 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T03:40:52,341 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/WALs/3a546af21488,34671,1732765249447 2024-11-28T03:40:52,343 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3a546af21488,34671,1732765249447] 2024-11-28T03:40:52,368 INFO [RS:0;3a546af21488:34671 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T03:40:52,381 INFO [RS:0;3a546af21488:34671 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T03:40:52,386 INFO [RS:0;3a546af21488:34671 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T03:40:52,386 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,387 INFO [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T03:40:52,393 INFO [RS:0;3a546af21488:34671 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T03:40:52,394 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,394 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,395 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,395 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,395 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,395 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,395 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3a546af21488:0, corePoolSize=2, maxPoolSize=2 2024-11-28T03:40:52,395 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,395 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,396 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,396 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,396 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,396 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:40:52,396 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3a546af21488:0, corePoolSize=3, maxPoolSize=3 2024-11-28T03:40:52,396 DEBUG [RS:0;3a546af21488:34671 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0, corePoolSize=3, maxPoolSize=3 2024-11-28T03:40:52,398 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,398 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,398 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,398 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,398 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,398 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,34671,1732765249447-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T03:40:52,419 INFO [RS:0;3a546af21488:34671 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T03:40:52,421 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,34671,1732765249447-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,421 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,421 INFO [RS:0;3a546af21488:34671 {}] regionserver.Replication(171): 3a546af21488,34671,1732765249447 started 2024-11-28T03:40:52,440 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:52,440 INFO [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1482): Serving as 3a546af21488,34671,1732765249447, RpcServer on 3a546af21488/172.17.0.2:34671, sessionid=0x1003c3721c70001 2024-11-28T03:40:52,441 DEBUG [RS:0;3a546af21488:34671 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T03:40:52,441 DEBUG [RS:0;3a546af21488:34671 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3a546af21488,34671,1732765249447 2024-11-28T03:40:52,441 DEBUG [RS:0;3a546af21488:34671 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3a546af21488,34671,1732765249447' 2024-11-28T03:40:52,441 DEBUG [RS:0;3a546af21488:34671 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T03:40:52,442 DEBUG [RS:0;3a546af21488:34671 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T03:40:52,443 DEBUG [RS:0;3a546af21488:34671 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T03:40:52,443 DEBUG [RS:0;3a546af21488:34671 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T03:40:52,443 DEBUG [RS:0;3a546af21488:34671 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3a546af21488,34671,1732765249447 2024-11-28T03:40:52,444 DEBUG [RS:0;3a546af21488:34671 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3a546af21488,34671,1732765249447' 2024-11-28T03:40:52,444 DEBUG [RS:0;3a546af21488:34671 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T03:40:52,444 DEBUG [RS:0;3a546af21488:34671 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T03:40:52,445 DEBUG [RS:0;3a546af21488:34671 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T03:40:52,445 INFO [RS:0;3a546af21488:34671 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T03:40:52,445 INFO [RS:0;3a546af21488:34671 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T03:40:52,551 INFO [RS:0;3a546af21488:34671 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T03:40:52,554 INFO [RS:0;3a546af21488:34671 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3a546af21488%2C34671%2C1732765249447, suffix=, logDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/WALs/3a546af21488,34671,1732765249447, archiveDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/oldWALs, maxLogs=32 2024-11-28T03:40:52,572 DEBUG [RS:0;3a546af21488:34671 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/WALs/3a546af21488,34671,1732765249447/3a546af21488%2C34671%2C1732765249447.1732765252557, exclude list is [], retry=0 2024-11-28T03:40:52,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35257,DS-90071d3e-5aaf-4155-97e6-98eee4eb70d9,DISK] 2024-11-28T03:40:52,580 INFO [RS:0;3a546af21488:34671 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/WALs/3a546af21488,34671,1732765249447/3a546af21488%2C34671%2C1732765249447.1732765252557 2024-11-28T03:40:52,580 DEBUG [RS:0;3a546af21488:34671 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39407:39407)] 2024-11-28T03:40:52,625 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T03:40:52,626 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72 2024-11-28T03:40:52,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741833_1009 (size=32) 2024-11-28T03:40:53,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:40:53,041 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T03:40:53,043 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T03:40:53,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:53,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:40:53,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T03:40:53,047 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T03:40:53,048 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:53,048 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:40:53,049 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T03:40:53,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T03:40:53,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:53,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:40:53,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T03:40:53,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T03:40:53,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:53,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:40:53,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T03:40:53,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740 2024-11-28T03:40:53,058 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740 2024-11-28T03:40:53,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T03:40:53,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T03:40:53,062 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:40:53,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T03:40:53,068 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:40:53,069 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65091269, jitterRate=-0.030064508318901062}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:40:53,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732765253038Initializing all the Stores at 1732765253040 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732765253040Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732765253041 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765253041Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732765253041Cleaning up temporary data from old regions at 1732765253061 (+20 ms)Region opened successfully at 1732765253071 (+10 ms) 2024-11-28T03:40:53,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T03:40:53,071 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T03:40:53,072 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T03:40:53,072 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T03:40:53,072 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T03:40:53,073 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T03:40:53,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732765253071Disabling compacts and flushes for region at 1732765253071Disabling writes for close at 1732765253072 (+1 ms)Writing region close event to WAL at 1732765253073 (+1 ms)Closed at 1732765253073 2024-11-28T03:40:53,076 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T03:40:53,077 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T03:40:53,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T03:40:53,092 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T03:40:53,095 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T03:40:53,248 DEBUG [3a546af21488:38327 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T03:40:53,260 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:40:53,266 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3a546af21488,34671,1732765249447, state=OPENING 2024-11-28T03:40:53,271 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T03:40:53,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:53,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:40:53,274 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T03:40:53,274 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T03:40:53,276 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T03:40:53,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:40:53,454 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T03:40:53,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59981, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T03:40:53,471 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T03:40:53,471 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T03:40:53,471 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T03:40:53,475 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3a546af21488%2C34671%2C1732765249447.meta, suffix=.meta, logDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/WALs/3a546af21488,34671,1732765249447, archiveDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/oldWALs, maxLogs=32 2024-11-28T03:40:53,491 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/WALs/3a546af21488,34671,1732765249447/3a546af21488%2C34671%2C1732765249447.meta.1732765253476.meta, exclude list is [], retry=0 2024-11-28T03:40:53,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35257,DS-90071d3e-5aaf-4155-97e6-98eee4eb70d9,DISK] 2024-11-28T03:40:53,498 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/WALs/3a546af21488,34671,1732765249447/3a546af21488%2C34671%2C1732765249447.meta.1732765253476.meta 2024-11-28T03:40:53,499 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39407:39407)] 2024-11-28T03:40:53,499 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:40:53,501 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T03:40:53,504 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T03:40:53,508 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T03:40:53,513 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T03:40:53,513 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:40:53,514 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T03:40:53,514 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T03:40:53,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T03:40:53,519 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T03:40:53,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:53,520 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:40:53,520 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T03:40:53,522 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T03:40:53,522 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:53,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:40:53,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T03:40:53,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T03:40:53,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:53,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:40:53,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T03:40:53,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T03:40:53,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:53,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:40:53,528 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T03:40:53,530 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740 2024-11-28T03:40:53,532 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740 2024-11-28T03:40:53,535 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T03:40:53,535 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T03:40:53,536 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:40:53,538 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T03:40:53,540 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63487623, jitterRate=-0.05396069586277008}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:40:53,540 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T03:40:53,542 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732765253514Writing region info on filesystem at 1732765253514Initializing all the Stores at 1732765253516 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732765253516Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732765253517 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765253517Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732765253517Cleaning up temporary data from old regions at 1732765253535 (+18 ms)Running coprocessor post-open hooks at 1732765253540 (+5 ms)Region opened successfully at 1732765253541 (+1 ms) 2024-11-28T03:40:53,549 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732765253445 2024-11-28T03:40:53,564 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T03:40:53,565 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T03:40:53,566 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:40:53,568 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3a546af21488,34671,1732765249447, state=OPEN 2024-11-28T03:40:53,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T03:40:53,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T03:40:53,574 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T03:40:53,574 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T03:40:53,574 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:53,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T03:40:53,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3a546af21488,34671,1732765249447 in 296 msec 2024-11-28T03:40:53,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T03:40:53,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 499 msec 2024-11-28T03:40:53,589 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T03:40:53,589 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T03:40:53,611 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:53,612 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:53,633 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:53,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38231, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:53,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5440 sec 2024-11-28T03:40:53,659 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732765253659, completionTime=-1 2024-11-28T03:40:53,663 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-28T03:40:53,663 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T03:40:53,701 INFO [master/3a546af21488:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-28T03:40:53,701 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732765313701 2024-11-28T03:40:53,702 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732765373702 2024-11-28T03:40:53,702 INFO [master/3a546af21488:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 38 msec 2024-11-28T03:40:53,705 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,38327,1732765248628-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:53,705 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,38327,1732765248628-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:53,706 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,38327,1732765248628-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:53,708 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3a546af21488:38327, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:53,708 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:53,709 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:53,715 DEBUG [master/3a546af21488:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T03:40:53,741 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.084sec 2024-11-28T03:40:53,743 INFO [master/3a546af21488:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T03:40:53,744 INFO [master/3a546af21488:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T03:40:53,746 INFO [master/3a546af21488:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T03:40:53,746 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T03:40:53,746 INFO [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T03:40:53,747 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,38327,1732765248628-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T03:40:53,748 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,38327,1732765248628-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T03:40:53,757 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T03:40:53,758 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T03:40:53,759 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,38327,1732765248628-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:40:53,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4de0fcb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:53,803 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T03:40:53,803 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T03:40:53,806 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:53,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:53,817 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305448db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:53,829 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:53,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f25d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:53,832 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:53,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:53,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:53,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3a546af21488,38327,1732765248628 2024-11-28T03:40:53,878 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=7469 2024-11-28T03:40:53,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T03:40:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3a546af21488,38327,1732765248628 2024-11-28T03:40:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@690ff3e4 2024-11-28T03:40:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T03:40:53,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T03:40:53,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T03:40:53,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:40:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T03:40:53,939 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:40:53,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-28T03:40:53,942 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:53,949 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:40:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T03:40:53,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741835_1011 (size=1061) 2024-11-28T03:40:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T03:40:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T03:40:54,365 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72 2024-11-28T03:40:54,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741836_1012 (size=53) 2024-11-28T03:40:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T03:40:54,775 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:40:54,776 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b7a40554a92e84435d4dc19dc444ec8c, disabling compactions & flushes 2024-11-28T03:40:54,776 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:54,776 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:54,776 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. after waiting 0 ms 2024-11-28T03:40:54,776 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:54,776 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:54,776 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b7a40554a92e84435d4dc19dc444ec8c: Waiting for close lock at 1732765254776Disabling compacts and flushes for region at 1732765254776Disabling writes for close at 1732765254776Writing region close event to WAL at 1732765254776Closed at 1732765254776 2024-11-28T03:40:54,778 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:40:54,784 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732765254779"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732765254779"}]},"ts":"1732765254779"} 2024-11-28T03:40:54,788 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:40:54,790 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:40:54,793 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765254790"}]},"ts":"1732765254790"} 2024-11-28T03:40:54,797 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T03:40:54,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a40554a92e84435d4dc19dc444ec8c, ASSIGN}] 2024-11-28T03:40:54,801 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a40554a92e84435d4dc19dc444ec8c, ASSIGN 2024-11-28T03:40:54,804 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a40554a92e84435d4dc19dc444ec8c, ASSIGN; state=OFFLINE, location=3a546af21488,34671,1732765249447; forceNewPlan=false, retain=false 2024-11-28T03:40:54,955 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b7a40554a92e84435d4dc19dc444ec8c, regionState=OPENING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:40:54,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a40554a92e84435d4dc19dc444ec8c, ASSIGN because future has completed 2024-11-28T03:40:54,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:40:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T03:40:55,122 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:55,122 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:40:55,123 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,123 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:40:55,123 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,123 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,126 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,129 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:40:55,130 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7a40554a92e84435d4dc19dc444ec8c columnFamilyName A 2024-11-28T03:40:55,130 DEBUG [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:55,131 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] regionserver.HStore(327): Store=b7a40554a92e84435d4dc19dc444ec8c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:40:55,131 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,133 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:40:55,134 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7a40554a92e84435d4dc19dc444ec8c columnFamilyName B 2024-11-28T03:40:55,134 DEBUG [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:55,135 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] regionserver.HStore(327): Store=b7a40554a92e84435d4dc19dc444ec8c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:40:55,135 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,137 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:40:55,138 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7a40554a92e84435d4dc19dc444ec8c columnFamilyName C 2024-11-28T03:40:55,138 DEBUG [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:40:55,139 INFO [StoreOpener-b7a40554a92e84435d4dc19dc444ec8c-1 {}] regionserver.HStore(327): Store=b7a40554a92e84435d4dc19dc444ec8c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:40:55,139 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:55,139 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,141 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,141 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,143 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,143 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,145 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:40:55,147 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,151 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:40:55,152 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b7a40554a92e84435d4dc19dc444ec8c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72516528, jitterRate=0.08058047294616699}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:40:55,152 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:55,153 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b7a40554a92e84435d4dc19dc444ec8c: Running coprocessor pre-open hook at 1732765255124Writing region info on filesystem at 1732765255124Initializing all the Stores at 1732765255125 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765255125Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765255126 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765255126Cleaning up temporary data from old regions at 1732765255143 (+17 ms)Running coprocessor post-open hooks at 1732765255152 (+9 ms)Region opened successfully at 1732765255153 (+1 ms) 2024-11-28T03:40:55,155 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., pid=6, masterSystemTime=1732765255115 2024-11-28T03:40:55,159 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:55,160 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:55,161 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b7a40554a92e84435d4dc19dc444ec8c, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:40:55,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:40:55,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T03:40:55,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 in 208 msec 2024-11-28T03:40:55,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T03:40:55,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a40554a92e84435d4dc19dc444ec8c, ASSIGN in 374 msec 2024-11-28T03:40:55,178 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:40:55,178 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765255178"}]},"ts":"1732765255178"} 2024-11-28T03:40:55,182 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T03:40:55,183 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:40:55,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2530 sec 2024-11-28T03:40:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T03:40:56,108 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:40:56,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f65897, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:56,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:56,115 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71374046, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:56,119 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3ecdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,121 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:56,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:56,128 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T03:40:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3a546af21488,38327,1732765248628 2024-11-28T03:40:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d0fb234 2024-11-28T03:40:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T03:40:56,135 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T03:40:56,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c8a4c89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:56,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:56,141 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec01134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:56,144 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b37f010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41658c86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:56,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:56,149 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38788dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:56,151 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@356cdf00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:56,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:56,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a084b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:56,160 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6b748f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ad1e082, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:56,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:56,168 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da77f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:56,171 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5530ffbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5760aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:56,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:56,175 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6830a8bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:56,177 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499c3205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d889989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:56,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:56,185 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5385c31b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:56,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:56,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:56,188 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:56,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f7ed9b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35c26788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:56,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:56,192 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:56,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:56,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:56,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce25cec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:56,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:56,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:56,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:56,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ac4719, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f79a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:56,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:56,198 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:56,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:56,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:56,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@252fb8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:56,200 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b76b53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ee0896, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:40:56,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:40:56,205 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:40:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:40:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:40:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf3278a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:40:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:40:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:40:56,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:40:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e46cb21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:40:56,217 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:56,217 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:56,218 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:56,218 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:56,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:56,219 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:56,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:56,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:56,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:56,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:56,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:56,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:56,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:56,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:56,227 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:40:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:40:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:40:56,235 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:56,236 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:56,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:40:56,241 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:40:56,242 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:40:56,244 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:56,245 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:56,245 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:56,246 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:40:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:56,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:56,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:56,249 DEBUG [Thread-154 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:40:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:40:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:40:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-28T03:40:56,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:40:56,252 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:40:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:40:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:40:56,255 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:40:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T03:40:56,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:40:56,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:40:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:56,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T03:40:56,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:40:56,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:56,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:40:56,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:56,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:40:56,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T03:40:56,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:56,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:56,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:56,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:56,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765316448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/3270cab6ae3049bb9649e40701cf9a93 is 50, key is test_row_0/A:col10/1732765256298/Put/seqid=0 2024-11-28T03:40:56,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765256436, max timestamp=1732765256468Num uniques 859; 2024-11-28T03:40:56,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:40:56,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765256436, max timestamp=1732765256468Num uniques 859; 2024-11-28T03:40:56,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:40:56,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765256436, max timestamp=1732765256468Num uniques 859; 2024-11-28T03:40:56,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741837_1013 (size=12001) 2024-11-28T03:40:56,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:40:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T03:40:56,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:56,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:56,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:56,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:56,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765316637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765316639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765316658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765316659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765316663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765316767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765316779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765316788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765316789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765316789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:56,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T03:40:56,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/3270cab6ae3049bb9649e40701cf9a93 2024-11-28T03:40:56,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:56,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:56,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:56,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:56,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:56,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765316987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:56,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765316987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:56,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765317010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765317010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765317017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/1288ffc549cb401e923165948ad5239e is 50, key is test_row_0/B:col10/1732765256298/Put/seqid=0 2024-11-28T03:40:57,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741838_1014 (size=12001) 2024-11-28T03:40:57,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/1288ffc549cb401e923165948ad5239e 2024-11-28T03:40:57,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:57,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/04b9f19a1bd24a9a9b3bd6cde1a6a281 is 50, key is test_row_0/C:col10/1732765256298/Put/seqid=0 2024-11-28T03:40:57,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741839_1015 (size=12001) 2024-11-28T03:40:57,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/04b9f19a1bd24a9a9b3bd6cde1a6a281 2024-11-28T03:40:57,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/3270cab6ae3049bb9649e40701cf9a93 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/3270cab6ae3049bb9649e40701cf9a93 2024-11-28T03:40:57,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/3270cab6ae3049bb9649e40701cf9a93, entries=150, sequenceid=14, filesize=11.7 K 2024-11-28T03:40:57,253 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-28T03:40:57,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/1288ffc549cb401e923165948ad5239e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1288ffc549cb401e923165948ad5239e 2024-11-28T03:40:57,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1288ffc549cb401e923165948ad5239e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-28T03:40:57,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/04b9f19a1bd24a9a9b3bd6cde1a6a281 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/04b9f19a1bd24a9a9b3bd6cde1a6a281 2024-11-28T03:40:57,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/04b9f19a1bd24a9a9b3bd6cde1a6a281, entries=150, sequenceid=14, filesize=11.7 K 2024-11-28T03:40:57,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:57,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:57,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for b7a40554a92e84435d4dc19dc444ec8c in 993ms, sequenceid=14, compaction requested=false 2024-11-28T03:40:57,314 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T03:40:57,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:40:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:57,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-28T03:40:57,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:40:57,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:40:57,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:40:57,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:40:57,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:40:57,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:40:57,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/cc34effed35d4b8091f5d3e7c31168ea is 50, key is test_row_0/A:col10/1732765257330/Put/seqid=0 2024-11-28T03:40:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T03:40:57,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741840_1016 (size=16681) 2024-11-28T03:40:57,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765317402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765317404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765317409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765317425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765317446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765317558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765317559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765317567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:57,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:57,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765317597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765317607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:57,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765317798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765317799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765317799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/cc34effed35d4b8091f5d3e7c31168ea 2024-11-28T03:40:57,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765317839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765317839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:57,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/1156b9c960c048b29230f4b0fc8780e5 is 50, key is test_row_0/B:col10/1732765257330/Put/seqid=0 2024-11-28T03:40:57,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741841_1017 (size=12001) 2024-11-28T03:40:57,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/1156b9c960c048b29230f4b0fc8780e5 2024-11-28T03:40:57,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:57,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:57,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:57,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:57,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/17d753e28429406d84ba56d23ad02995 is 50, key is test_row_0/C:col10/1732765257330/Put/seqid=0 2024-11-28T03:40:57,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741842_1018 (size=12001) 2024-11-28T03:40:57,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/17d753e28429406d84ba56d23ad02995 2024-11-28T03:40:57,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/cc34effed35d4b8091f5d3e7c31168ea as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc34effed35d4b8091f5d3e7c31168ea 2024-11-28T03:40:58,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc34effed35d4b8091f5d3e7c31168ea, entries=250, sequenceid=50, filesize=16.3 K 2024-11-28T03:40:58,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/1156b9c960c048b29230f4b0fc8780e5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1156b9c960c048b29230f4b0fc8780e5 2024-11-28T03:40:58,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1156b9c960c048b29230f4b0fc8780e5, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T03:40:58,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/17d753e28429406d84ba56d23ad02995 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/17d753e28429406d84ba56d23ad02995 2024-11-28T03:40:58,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/17d753e28429406d84ba56d23ad02995, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T03:40:58,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:58,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:58,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for b7a40554a92e84435d4dc19dc444ec8c in 761ms, sequenceid=50, compaction requested=false 2024-11-28T03:40:58,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:40:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:58,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T03:40:58,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:40:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:40:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:40:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:58,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/73fa8883946f4fa8a45bac0ad92710fc is 50, key is test_row_0/A:col10/1732765258131/Put/seqid=0 2024-11-28T03:40:58,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741843_1019 (size=14341) 2024-11-28T03:40:58,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765318275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765318276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765318279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765318286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765318289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,361 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T03:40:58,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:58,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:58,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T03:40:58,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765318398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765318399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765318399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765318408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765318408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/73fa8883946f4fa8a45bac0ad92710fc 2024-11-28T03:40:58,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/ff7d4707b1264258b604906a03126654 is 50, key is test_row_0/B:col10/1732765258131/Put/seqid=0 2024-11-28T03:40:58,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765318617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765318618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765318627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765318627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765318629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:58,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741844_1020 (size=12001) 2024-11-28T03:40:58,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/ff7d4707b1264258b604906a03126654 2024-11-28T03:40:58,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/0ecd996e7775440f9d13b20e0d56a7b2 is 50, key is test_row_0/C:col10/1732765258131/Put/seqid=0 2024-11-28T03:40:58,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:40:58,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741845_1021 (size=12001) 2024-11-28T03:40:58,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/0ecd996e7775440f9d13b20e0d56a7b2 2024-11-28T03:40:58,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/73fa8883946f4fa8a45bac0ad92710fc as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/73fa8883946f4fa8a45bac0ad92710fc 2024-11-28T03:40:58,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/73fa8883946f4fa8a45bac0ad92710fc, entries=200, sequenceid=64, filesize=14.0 K 2024-11-28T03:40:58,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/ff7d4707b1264258b604906a03126654 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ff7d4707b1264258b604906a03126654 2024-11-28T03:40:58,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ff7d4707b1264258b604906a03126654, entries=150, sequenceid=64, filesize=11.7 K 2024-11-28T03:40:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/0ecd996e7775440f9d13b20e0d56a7b2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0ecd996e7775440f9d13b20e0d56a7b2 2024-11-28T03:40:58,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0ecd996e7775440f9d13b20e0d56a7b2, entries=150, sequenceid=64, filesize=11.7 K 2024-11-28T03:40:58,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b7a40554a92e84435d4dc19dc444ec8c in 691ms, sequenceid=64, compaction requested=true 2024-11-28T03:40:58,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:40:58,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:40:58,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:40:58,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:40:58,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:40:58,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:40:58,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:40:58,846 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:40:58,846 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:40:58,851 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:40:58,853 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:40:58,853 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,854 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1288ffc549cb401e923165948ad5239e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1156b9c960c048b29230f4b0fc8780e5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ff7d4707b1264258b604906a03126654] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=35.2 K 2024-11-28T03:40:58,855 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:40:58,855 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:40:58,856 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,856 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/3270cab6ae3049bb9649e40701cf9a93, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc34effed35d4b8091f5d3e7c31168ea, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/73fa8883946f4fa8a45bac0ad92710fc] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=42.0 K 2024-11-28T03:40:58,857 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1288ffc549cb401e923165948ad5239e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732765256270 2024-11-28T03:40:58,858 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3270cab6ae3049bb9649e40701cf9a93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732765256270 2024-11-28T03:40:58,858 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1156b9c960c048b29230f4b0fc8780e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732765256635 2024-11-28T03:40:58,859 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ff7d4707b1264258b604906a03126654, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732765257415 2024-11-28T03:40:58,860 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc34effed35d4b8091f5d3e7c31168ea, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732765256635 2024-11-28T03:40:58,864 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73fa8883946f4fa8a45bac0ad92710fc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732765257415 2024-11-28T03:40:58,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T03:40:58,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:58,872 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:40:58,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:40:58,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:58,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:40:58,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:58,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:40:58,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:58,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/8f54f6801f7f4206852880067ea1472f is 50, key is test_row_0/A:col10/1732765258274/Put/seqid=0 2024-11-28T03:40:58,912 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:40:58,914 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/0ff513ecb820405e8fb69c87ba875b9e is 50, key is test_row_0/B:col10/1732765258131/Put/seqid=0 2024-11-28T03:40:58,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741846_1022 (size=12001) 2024-11-28T03:40:58,930 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/8f54f6801f7f4206852880067ea1472f 2024-11-28T03:40:58,951 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#11 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:40:58,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:40:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:58,956 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7b0d99fb7fec4b2daa4891d6834bae91 is 50, key is test_row_0/A:col10/1732765258131/Put/seqid=0 2024-11-28T03:40:58,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741847_1023 (size=12104) 2024-11-28T03:40:58,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/da9f5cdec8a84b0eb12bd4c136d56a07 is 50, key is test_row_0/B:col10/1732765258274/Put/seqid=0 2024-11-28T03:40:58,993 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/0ff513ecb820405e8fb69c87ba875b9e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/0ff513ecb820405e8fb69c87ba875b9e 2024-11-28T03:40:59,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765319000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,027 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:40:59,030 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into 0ff513ecb820405e8fb69c87ba875b9e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:40:59,030 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:40:59,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765319003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741848_1024 (size=12104) 2024-11-28T03:40:59,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765319011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765319013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765319020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,046 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=13, startTime=1732765258843; duration=0sec 2024-11-28T03:40:59,047 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:40:59,047 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:40:59,047 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:40:59,053 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:40:59,053 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:40:59,053 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:59,053 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/04b9f19a1bd24a9a9b3bd6cde1a6a281, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/17d753e28429406d84ba56d23ad02995, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0ecd996e7775440f9d13b20e0d56a7b2] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=35.2 K 2024-11-28T03:40:59,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741849_1025 (size=12001) 2024-11-28T03:40:59,055 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7b0d99fb7fec4b2daa4891d6834bae91 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7b0d99fb7fec4b2daa4891d6834bae91 2024-11-28T03:40:59,056 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/da9f5cdec8a84b0eb12bd4c136d56a07 2024-11-28T03:40:59,057 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b9f19a1bd24a9a9b3bd6cde1a6a281, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732765256270 2024-11-28T03:40:59,059 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 17d753e28429406d84ba56d23ad02995, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732765256635 2024-11-28T03:40:59,064 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ecd996e7775440f9d13b20e0d56a7b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732765257415 2024-11-28T03:40:59,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/69282aee0fc3434d99cf91af777f88c3 is 50, key is test_row_0/C:col10/1732765258274/Put/seqid=0 2024-11-28T03:40:59,081 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into 7b0d99fb7fec4b2daa4891d6834bae91(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:40:59,081 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:40:59,081 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=13, startTime=1732765258826; duration=0sec 2024-11-28T03:40:59,083 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:40:59,084 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:40:59,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741850_1026 (size=12001) 2024-11-28T03:40:59,125 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#14 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:40:59,126 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/54205189e8354dd1a21eaea032689b7a is 50, key is test_row_0/C:col10/1732765258131/Put/seqid=0 2024-11-28T03:40:59,127 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/69282aee0fc3434d99cf91af777f88c3 2024-11-28T03:40:59,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765319127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/8f54f6801f7f4206852880067ea1472f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8f54f6801f7f4206852880067ea1472f 2024-11-28T03:40:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765319138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765319148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,154 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8f54f6801f7f4206852880067ea1472f, entries=150, sequenceid=82, filesize=11.7 K 2024-11-28T03:40:59,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/da9f5cdec8a84b0eb12bd4c136d56a07 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da9f5cdec8a84b0eb12bd4c136d56a07 2024-11-28T03:40:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765319148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765319149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,169 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T03:40:59,170 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da9f5cdec8a84b0eb12bd4c136d56a07, entries=150, sequenceid=82, filesize=11.7 K 2024-11-28T03:40:59,170 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-28T03:40:59,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/69282aee0fc3434d99cf91af777f88c3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/69282aee0fc3434d99cf91af777f88c3 2024-11-28T03:40:59,173 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T03:40:59,173 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-28T03:40:59,173 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-28T03:40:59,173 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-28T03:40:59,175 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T03:40:59,175 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T03:40:59,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741851_1027 (size=12104) 2024-11-28T03:40:59,185 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/69282aee0fc3434d99cf91af777f88c3, entries=150, sequenceid=82, filesize=11.7 K 2024-11-28T03:40:59,187 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b7a40554a92e84435d4dc19dc444ec8c in 315ms, sequenceid=82, compaction requested=false 2024-11-28T03:40:59,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:40:59,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:40:59,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-28T03:40:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-28T03:40:59,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T03:40:59,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9330 sec 2024-11-28T03:40:59,198 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.9510 sec 2024-11-28T03:40:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:40:59,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T03:40:59,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:40:59,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:59,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:40:59,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:59,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:40:59,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:40:59,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/a24e455d30f1457c89bc69169dbf43c2 is 50, key is test_row_0/A:col10/1732765259002/Put/seqid=0 2024-11-28T03:40:59,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741852_1028 (size=14341) 2024-11-28T03:40:59,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/a24e455d30f1457c89bc69169dbf43c2 2024-11-28T03:40:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765319398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765319415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765319416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765319419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765319418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/03b7843892eb488bb8c7afa3be62cca3 is 50, key is test_row_0/B:col10/1732765259002/Put/seqid=0 2024-11-28T03:40:59,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741853_1029 (size=12001) 2024-11-28T03:40:59,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765319527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765319536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765319537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765319537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765319537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,595 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/54205189e8354dd1a21eaea032689b7a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/54205189e8354dd1a21eaea032689b7a 2024-11-28T03:40:59,714 INFO [AsyncFSWAL-0-hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72-prefix:3a546af21488,34671,1732765249447 {}] wal.AbstractFSWAL(1368): Slow sync cost: 100 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35257,DS-90071d3e-5aaf-4155-97e6-98eee4eb70d9,DISK]] 2024-11-28T03:40:59,715 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into 54205189e8354dd1a21eaea032689b7a(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:40:59,715 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:40:59,715 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=13, startTime=1732765258843; duration=0sec 2024-11-28T03:40:59,715 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:40:59,715 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:40:59,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765319758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765319759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765319759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765319759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:40:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765319767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:40:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:40:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:40:59,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/03b7843892eb488bb8c7afa3be62cca3 2024-11-28T03:40:59,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/5c6621a6d87f4346becee382e2d20620 is 50, key is test_row_0/C:col10/1732765259002/Put/seqid=0 2024-11-28T03:40:59,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741854_1030 (size=12001) 2024-11-28T03:41:00,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765320067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765320067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765320068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765320068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765320076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/5c6621a6d87f4346becee382e2d20620 2024-11-28T03:41:00,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/a24e455d30f1457c89bc69169dbf43c2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a24e455d30f1457c89bc69169dbf43c2 2024-11-28T03:41:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T03:41:00,417 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:00,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:00,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a24e455d30f1457c89bc69169dbf43c2, entries=200, sequenceid=99, filesize=14.0 K 2024-11-28T03:41:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-28T03:41:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T03:41:00,424 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:00,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/03b7843892eb488bb8c7afa3be62cca3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/03b7843892eb488bb8c7afa3be62cca3 2024-11-28T03:41:00,426 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:00,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:00,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/03b7843892eb488bb8c7afa3be62cca3, entries=150, sequenceid=99, filesize=11.7 K 2024-11-28T03:41:00,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/5c6621a6d87f4346becee382e2d20620 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5c6621a6d87f4346becee382e2d20620 2024-11-28T03:41:00,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5c6621a6d87f4346becee382e2d20620, entries=150, sequenceid=99, filesize=11.7 K 2024-11-28T03:41:00,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b7a40554a92e84435d4dc19dc444ec8c in 1116ms, sequenceid=99, compaction requested=true 2024-11-28T03:41:00,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:00,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:00,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:00,469 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:00,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:00,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:00,469 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:00,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:00,471 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:00,471 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:00,471 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:00,471 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7b0d99fb7fec4b2daa4891d6834bae91, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8f54f6801f7f4206852880067ea1472f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a24e455d30f1457c89bc69169dbf43c2] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=37.5 K 2024-11-28T03:41:00,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:00,472 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:00,472 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:00,472 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:00,472 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/0ff513ecb820405e8fb69c87ba875b9e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da9f5cdec8a84b0eb12bd4c136d56a07, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/03b7843892eb488bb8c7afa3be62cca3] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=35.3 K 2024-11-28T03:41:00,472 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0d99fb7fec4b2daa4891d6834bae91, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732765257415 2024-11-28T03:41:00,473 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff513ecb820405e8fb69c87ba875b9e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732765257415 2024-11-28T03:41:00,473 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting da9f5cdec8a84b0eb12bd4c136d56a07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732765258271 2024-11-28T03:41:00,474 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f54f6801f7f4206852880067ea1472f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732765258271 2024-11-28T03:41:00,474 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 03b7843892eb488bb8c7afa3be62cca3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732765259002 2024-11-28T03:41:00,475 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a24e455d30f1457c89bc69169dbf43c2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732765259000 2024-11-28T03:41:00,494 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:00,495 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/a217043499114ab6a9fe41c955f52fb1 is 50, key is test_row_0/B:col10/1732765259002/Put/seqid=0 2024-11-28T03:41:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T03:41:00,571 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:00,573 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/8a628bbeef5f48a58b7840e61fd6c819 is 50, key is test_row_0/A:col10/1732765259002/Put/seqid=0 2024-11-28T03:41:00,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T03:41:00,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:00,581 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T03:41:00,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:00,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:00,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:00,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:00,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:00,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:00,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:00,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741856_1032 (size=12207) 2024-11-28T03:41:00,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/4e2d02c73c9e410dbadf6552152fb5a1 is 50, key is test_row_0/A:col10/1732765259410/Put/seqid=0 2024-11-28T03:41:00,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741857_1033 (size=12001) 2024-11-28T03:41:00,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741855_1031 (size=12207) 2024-11-28T03:41:00,614 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/4e2d02c73c9e410dbadf6552152fb5a1 2024-11-28T03:41:00,623 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/8a628bbeef5f48a58b7840e61fd6c819 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8a628bbeef5f48a58b7840e61fd6c819 2024-11-28T03:41:00,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/c746e3209d324c1881b221b3a0b21e3c is 50, key is test_row_0/B:col10/1732765259410/Put/seqid=0 2024-11-28T03:41:00,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765320631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765320631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765320635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,655 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into 8a628bbeef5f48a58b7840e61fd6c819(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:00,656 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:00,656 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=13, startTime=1732765260468; duration=0sec 2024-11-28T03:41:00,656 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:00,656 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:00,656 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:00,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765320648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,659 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:00,659 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,659 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765320652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,659 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/54205189e8354dd1a21eaea032689b7a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/69282aee0fc3434d99cf91af777f88c3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5c6621a6d87f4346becee382e2d20620] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=35.3 K 2024-11-28T03:41:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,660 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54205189e8354dd1a21eaea032689b7a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732765257415 2024-11-28T03:41:00,661 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69282aee0fc3434d99cf91af777f88c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732765258271 2024-11-28T03:41:00,662 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c6621a6d87f4346becee382e2d20620, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732765259002 2024-11-28T03:41:00,695 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:00,696 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/32b053977ad144918d448a757af85bf6 is 50, key is test_row_0/C:col10/1732765259002/Put/seqid=0 2024-11-28T03:41:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T03:41:00,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741858_1034 (size=12001) 2024-11-28T03:41:00,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765320757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765320757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765320757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765320768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765320772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741859_1035 (size=12207) 2024-11-28T03:41:00,810 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/32b053977ad144918d448a757af85bf6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/32b053977ad144918d448a757af85bf6 2024-11-28T03:41:00,825 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into 32b053977ad144918d448a757af85bf6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:00,825 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:00,825 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=13, startTime=1732765260470; duration=0sec 2024-11-28T03:41:00,825 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:00,825 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:00,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765320967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765320968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765320976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765320977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765320977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,026 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/a217043499114ab6a9fe41c955f52fb1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a217043499114ab6a9fe41c955f52fb1 2024-11-28T03:41:01,041 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into a217043499114ab6a9fe41c955f52fb1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:01,041 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:01,041 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=13, startTime=1732765260469; duration=0sec 2024-11-28T03:41:01,041 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:01,041 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T03:41:01,155 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/c746e3209d324c1881b221b3a0b21e3c 2024-11-28T03:41:01,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/e29bf4f302374362af566a3be94624fa is 50, key is test_row_0/C:col10/1732765259410/Put/seqid=0 2024-11-28T03:41:01,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741860_1036 (size=12001) 2024-11-28T03:41:01,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765321277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765321277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765321311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765321311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765321318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T03:41:01,594 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/e29bf4f302374362af566a3be94624fa 2024-11-28T03:41:01,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/4e2d02c73c9e410dbadf6552152fb5a1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4e2d02c73c9e410dbadf6552152fb5a1 2024-11-28T03:41:01,618 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4e2d02c73c9e410dbadf6552152fb5a1, entries=150, sequenceid=117, filesize=11.7 K 2024-11-28T03:41:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/c746e3209d324c1881b221b3a0b21e3c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/c746e3209d324c1881b221b3a0b21e3c 2024-11-28T03:41:01,631 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/c746e3209d324c1881b221b3a0b21e3c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-28T03:41:01,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/e29bf4f302374362af566a3be94624fa as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e29bf4f302374362af566a3be94624fa 2024-11-28T03:41:01,650 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e29bf4f302374362af566a3be94624fa, entries=150, sequenceid=117, filesize=11.7 K 2024-11-28T03:41:01,654 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for b7a40554a92e84435d4dc19dc444ec8c in 1073ms, sequenceid=117, compaction requested=false 2024-11-28T03:41:01,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:01,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:01,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-28T03:41:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-28T03:41:01,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-28T03:41:01,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2320 sec 2024-11-28T03:41:01,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2440 sec 2024-11-28T03:41:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:01,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T03:41:01,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:01,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:01,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:01,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:01,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:01,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:01,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/ee2ca06289e74abf86fbabf466ee9a3e is 50, key is test_row_0/A:col10/1732765261789/Put/seqid=0 2024-11-28T03:41:01,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741861_1037 (size=9757) 2024-11-28T03:41:01,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/ee2ca06289e74abf86fbabf466ee9a3e 2024-11-28T03:41:01,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/6925bca5adaa44c88975da79da454cbe is 50, key is test_row_0/B:col10/1732765261789/Put/seqid=0 2024-11-28T03:41:01,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765321838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765321839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765321847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765321850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765321850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741862_1038 (size=9757) 2024-11-28T03:41:01,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/6925bca5adaa44c88975da79da454cbe 2024-11-28T03:41:01,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/12d3d4b79113485f8511ce944f04725f is 50, key is test_row_0/C:col10/1732765261789/Put/seqid=0 2024-11-28T03:41:01,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741863_1039 (size=9757) 2024-11-28T03:41:01,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765321957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765321957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765321966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765321967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765321967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765322166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765322167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765322177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765322177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765322178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/12d3d4b79113485f8511ce944f04725f 2024-11-28T03:41:02,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/ee2ca06289e74abf86fbabf466ee9a3e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/ee2ca06289e74abf86fbabf466ee9a3e 2024-11-28T03:41:02,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/ee2ca06289e74abf86fbabf466ee9a3e, entries=100, sequenceid=138, filesize=9.5 K 2024-11-28T03:41:02,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/6925bca5adaa44c88975da79da454cbe as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6925bca5adaa44c88975da79da454cbe 2024-11-28T03:41:02,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6925bca5adaa44c88975da79da454cbe, entries=100, sequenceid=138, filesize=9.5 K 2024-11-28T03:41:02,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/12d3d4b79113485f8511ce944f04725f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/12d3d4b79113485f8511ce944f04725f 2024-11-28T03:41:02,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/12d3d4b79113485f8511ce944f04725f, entries=100, sequenceid=138, filesize=9.5 K 2024-11-28T03:41:02,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b7a40554a92e84435d4dc19dc444ec8c in 590ms, sequenceid=138, compaction requested=true 2024-11-28T03:41:02,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:02,381 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:02,383 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:02,383 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:02,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:02,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:02,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:02,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:02,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:02,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:02,384 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:02,384 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:02,385 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8a628bbeef5f48a58b7840e61fd6c819, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4e2d02c73c9e410dbadf6552152fb5a1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/ee2ca06289e74abf86fbabf466ee9a3e] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=33.2 K 2024-11-28T03:41:02,386 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a628bbeef5f48a58b7840e61fd6c819, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732765259002 2024-11-28T03:41:02,387 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e2d02c73c9e410dbadf6552152fb5a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732765259397 2024-11-28T03:41:02,388 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:02,388 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:02,388 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:02,388 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a217043499114ab6a9fe41c955f52fb1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/c746e3209d324c1881b221b3a0b21e3c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6925bca5adaa44c88975da79da454cbe] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=33.2 K 2024-11-28T03:41:02,389 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ee2ca06289e74abf86fbabf466ee9a3e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732765261789 2024-11-28T03:41:02,390 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a217043499114ab6a9fe41c955f52fb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732765259002 2024-11-28T03:41:02,394 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c746e3209d324c1881b221b3a0b21e3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732765259397 2024-11-28T03:41:02,395 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6925bca5adaa44c88975da79da454cbe, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732765261789 2024-11-28T03:41:02,416 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:02,417 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:02,417 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/71e624d104b341fda559171e38572fb1 is 50, key is test_row_0/B:col10/1732765261789/Put/seqid=0 2024-11-28T03:41:02,417 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/521724a408494ea08d19411bdedbc58e is 50, key is test_row_0/A:col10/1732765261789/Put/seqid=0 2024-11-28T03:41:02,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741864_1040 (size=12409) 2024-11-28T03:41:02,461 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/71e624d104b341fda559171e38572fb1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/71e624d104b341fda559171e38572fb1 2024-11-28T03:41:02,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741865_1041 (size=12409) 2024-11-28T03:41:02,480 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into 71e624d104b341fda559171e38572fb1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:02,481 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:02,481 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=13, startTime=1732765262384; duration=0sec 2024-11-28T03:41:02,481 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:02,481 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:02,481 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:02,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T03:41:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:02,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:02,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:02,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:02,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:02,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:02,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:02,487 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:02,487 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:02,487 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:02,487 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/32b053977ad144918d448a757af85bf6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e29bf4f302374362af566a3be94624fa, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/12d3d4b79113485f8511ce944f04725f] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=33.2 K 2024-11-28T03:41:02,488 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32b053977ad144918d448a757af85bf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732765259002 2024-11-28T03:41:02,489 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e29bf4f302374362af566a3be94624fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732765259397 2024-11-28T03:41:02,490 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d3d4b79113485f8511ce944f04725f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732765261789 2024-11-28T03:41:02,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/66c8cfaeffb44ee2827e714d90131ad1 is 50, key is test_row_0/A:col10/1732765261835/Put/seqid=0 2024-11-28T03:41:02,510 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:02,511 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/28358edcdcbf41b6b6f4b8b96b9d68bc is 50, key is test_row_0/C:col10/1732765261789/Put/seqid=0 2024-11-28T03:41:02,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765322524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765322524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765322527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765322531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765322533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741867_1043 (size=16931) 2024-11-28T03:41:02,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741866_1042 (size=12409) 2024-11-28T03:41:02,565 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/28358edcdcbf41b6b6f4b8b96b9d68bc as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/28358edcdcbf41b6b6f4b8b96b9d68bc 2024-11-28T03:41:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T03:41:02,567 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:02,568 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-28T03:41:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T03:41:02,572 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:02,574 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:02,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:02,578 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into 28358edcdcbf41b6b6f4b8b96b9d68bc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:02,578 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:02,578 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=13, startTime=1732765262384; duration=0sec 2024-11-28T03:41:02,579 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:02,580 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:02,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765322647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765322647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765322648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765322648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765322651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T03:41:02,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T03:41:02,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:02,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:02,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:02,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:02,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:02,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765322858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765322858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765322858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765322859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765322859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:02,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T03:41:02,887 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/521724a408494ea08d19411bdedbc58e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/521724a408494ea08d19411bdedbc58e 2024-11-28T03:41:02,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T03:41:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:02,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:02,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:02,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:02,904 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into 521724a408494ea08d19411bdedbc58e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:02,904 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:02,904 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=13, startTime=1732765262381; duration=0sec 2024-11-28T03:41:02,904 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:02,904 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:02,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/66c8cfaeffb44ee2827e714d90131ad1 2024-11-28T03:41:02,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/7399a14b984248e187b2aac60bf6631a is 50, key is test_row_0/B:col10/1732765261835/Put/seqid=0 2024-11-28T03:41:02,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741868_1044 (size=12151) 2024-11-28T03:41:02,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/7399a14b984248e187b2aac60bf6631a 2024-11-28T03:41:03,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/5510e3577fe44e3fb1d595dccb680b7e is 50, key is test_row_0/C:col10/1732765261835/Put/seqid=0 2024-11-28T03:41:03,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741869_1045 (size=12151) 2024-11-28T03:41:03,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T03:41:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:03,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:03,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765323166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765323166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765323167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765323167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765323176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T03:41:03,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T03:41:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:03,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:03,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T03:41:03,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:03,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:03,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:03,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:03,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:03,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/5510e3577fe44e3fb1d595dccb680b7e 2024-11-28T03:41:03,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/66c8cfaeffb44ee2827e714d90131ad1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/66c8cfaeffb44ee2827e714d90131ad1 2024-11-28T03:41:03,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/66c8cfaeffb44ee2827e714d90131ad1, entries=250, sequenceid=155, filesize=16.5 K 2024-11-28T03:41:03,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/7399a14b984248e187b2aac60bf6631a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7399a14b984248e187b2aac60bf6631a 2024-11-28T03:41:03,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7399a14b984248e187b2aac60bf6631a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-28T03:41:03,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/5510e3577fe44e3fb1d595dccb680b7e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5510e3577fe44e3fb1d595dccb680b7e 2024-11-28T03:41:03,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5510e3577fe44e3fb1d595dccb680b7e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-28T03:41:03,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b7a40554a92e84435d4dc19dc444ec8c in 1007ms, sequenceid=155, compaction requested=false 2024-11-28T03:41:03,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:03,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T03:41:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:03,525 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T03:41:03,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:03,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:03,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:03,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:03,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:03,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:03,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/43bf11a9ddee4e50b6cbc3fdf93e1bf3 is 50, key is test_row_0/A:col10/1732765262528/Put/seqid=0 2024-11-28T03:41:03,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741870_1046 (size=12151) 2024-11-28T03:41:03,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T03:41:03,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765323708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765323710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765323721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765323724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765323727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765323826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765323837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765323837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765323837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765323837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:03,965 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/43bf11a9ddee4e50b6cbc3fdf93e1bf3 2024-11-28T03:41:03,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/279c6526c1fa4360994c4f12f93881a0 is 50, key is test_row_0/B:col10/1732765262528/Put/seqid=0 2024-11-28T03:41:03,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741871_1047 (size=12151) 2024-11-28T03:41:03,987 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/279c6526c1fa4360994c4f12f93881a0 2024-11-28T03:41:04,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/0bea626a584148899c94534065519f76 is 50, key is test_row_0/C:col10/1732765262528/Put/seqid=0 2024-11-28T03:41:04,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741872_1048 (size=12151) 2024-11-28T03:41:04,010 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/0bea626a584148899c94534065519f76 2024-11-28T03:41:04,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/43bf11a9ddee4e50b6cbc3fdf93e1bf3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/43bf11a9ddee4e50b6cbc3fdf93e1bf3 2024-11-28T03:41:04,036 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/43bf11a9ddee4e50b6cbc3fdf93e1bf3, entries=150, sequenceid=173, filesize=11.9 K 2024-11-28T03:41:04,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/279c6526c1fa4360994c4f12f93881a0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/279c6526c1fa4360994c4f12f93881a0 2024-11-28T03:41:04,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765324036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,047 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/279c6526c1fa4360994c4f12f93881a0, entries=150, sequenceid=173, filesize=11.9 K 2024-11-28T03:41:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/0bea626a584148899c94534065519f76 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0bea626a584148899c94534065519f76 2024-11-28T03:41:04,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765324047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765324047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,060 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0bea626a584148899c94534065519f76, entries=150, sequenceid=173, filesize=11.9 K 2024-11-28T03:41:04,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765324058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,062 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for b7a40554a92e84435d4dc19dc444ec8c in 536ms, sequenceid=173, compaction requested=true 2024-11-28T03:41:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-28T03:41:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-28T03:41:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:04,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:41:04,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:04,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:04,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:04,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:04,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:04,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:04,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-28T03:41:04,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4900 sec 2024-11-28T03:41:04,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5010 sec 2024-11-28T03:41:04,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7ed4298656364e41841f646d8c4eeb73 is 50, key is test_row_0/A:col10/1732765264064/Put/seqid=0 2024-11-28T03:41:04,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741873_1049 (size=14541) 2024-11-28T03:41:04,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7ed4298656364e41841f646d8c4eeb73 2024-11-28T03:41:04,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/00d75a73f08f4ef1ac30f64aab9f159e is 50, key is test_row_0/B:col10/1732765264064/Put/seqid=0 2024-11-28T03:41:04,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741874_1050 (size=12151) 2024-11-28T03:41:04,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765324159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765324267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765324357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765324366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765324367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765324367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765324477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/00d75a73f08f4ef1ac30f64aab9f159e 2024-11-28T03:41:04,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/d4012def3064408fadb9656f7c120639 is 50, key is test_row_0/C:col10/1732765264064/Put/seqid=0 2024-11-28T03:41:04,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741875_1051 (size=12151) 2024-11-28T03:41:04,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/d4012def3064408fadb9656f7c120639 2024-11-28T03:41:04,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7ed4298656364e41841f646d8c4eeb73 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7ed4298656364e41841f646d8c4eeb73 2024-11-28T03:41:04,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7ed4298656364e41841f646d8c4eeb73, entries=200, sequenceid=191, filesize=14.2 K 2024-11-28T03:41:04,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/00d75a73f08f4ef1ac30f64aab9f159e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/00d75a73f08f4ef1ac30f64aab9f159e 2024-11-28T03:41:04,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/00d75a73f08f4ef1ac30f64aab9f159e, entries=150, sequenceid=191, filesize=11.9 K 2024-11-28T03:41:04,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/d4012def3064408fadb9656f7c120639 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d4012def3064408fadb9656f7c120639 2024-11-28T03:41:04,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d4012def3064408fadb9656f7c120639, entries=150, sequenceid=191, filesize=11.9 K 2024-11-28T03:41:04,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b7a40554a92e84435d4dc19dc444ec8c in 550ms, sequenceid=191, compaction requested=true 2024-11-28T03:41:04,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:04,617 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:04,617 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:04,620 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56032 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:04,620 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:04,620 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:04,620 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/521724a408494ea08d19411bdedbc58e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/66c8cfaeffb44ee2827e714d90131ad1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/43bf11a9ddee4e50b6cbc3fdf93e1bf3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7ed4298656364e41841f646d8c4eeb73] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=54.7 K 2024-11-28T03:41:04,621 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:04,621 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:04,621 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 521724a408494ea08d19411bdedbc58e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732765259413 2024-11-28T03:41:04,621 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:04,621 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/71e624d104b341fda559171e38572fb1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7399a14b984248e187b2aac60bf6631a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/279c6526c1fa4360994c4f12f93881a0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/00d75a73f08f4ef1ac30f64aab9f159e] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=47.7 K 2024-11-28T03:41:04,622 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66c8cfaeffb44ee2827e714d90131ad1, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732765261834 2024-11-28T03:41:04,622 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 71e624d104b341fda559171e38572fb1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732765259413 2024-11-28T03:41:04,622 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43bf11a9ddee4e50b6cbc3fdf93e1bf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732765262523 2024-11-28T03:41:04,623 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7399a14b984248e187b2aac60bf6631a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732765261835 2024-11-28T03:41:04,623 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ed4298656364e41841f646d8c4eeb73, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732765263707 2024-11-28T03:41:04,624 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 279c6526c1fa4360994c4f12f93881a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732765262523 2024-11-28T03:41:04,625 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 00d75a73f08f4ef1ac30f64aab9f159e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732765263721 2024-11-28T03:41:04,650 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:04,651 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:04,651 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/2d8f849e807241c3b37b51ec6710ea87 is 50, key is test_row_0/A:col10/1732765264064/Put/seqid=0 2024-11-28T03:41:04,652 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/7552475f542a4a1c95312412280aaba7 is 50, key is test_row_0/B:col10/1732765264064/Put/seqid=0 2024-11-28T03:41:04,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741876_1052 (size=12595) 2024-11-28T03:41:04,675 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/2d8f849e807241c3b37b51ec6710ea87 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/2d8f849e807241c3b37b51ec6710ea87 2024-11-28T03:41:04,686 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into 2d8f849e807241c3b37b51ec6710ea87(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:04,686 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:04,687 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=12, startTime=1732765264616; duration=0sec 2024-11-28T03:41:04,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741877_1053 (size=12595) 2024-11-28T03:41:04,690 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:04,690 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:04,690 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:04,693 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:04,693 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:04,693 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:04,693 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/28358edcdcbf41b6b6f4b8b96b9d68bc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5510e3577fe44e3fb1d595dccb680b7e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0bea626a584148899c94534065519f76, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d4012def3064408fadb9656f7c120639] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=47.7 K 2024-11-28T03:41:04,694 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28358edcdcbf41b6b6f4b8b96b9d68bc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732765259413 2024-11-28T03:41:04,694 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5510e3577fe44e3fb1d595dccb680b7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732765261835 2024-11-28T03:41:04,695 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bea626a584148899c94534065519f76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732765262523 2024-11-28T03:41:04,696 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4012def3064408fadb9656f7c120639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732765263721 2024-11-28T03:41:04,715 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:04,716 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/e62619ff9ec34cee9b5e7d199203ac28 is 50, key is test_row_0/C:col10/1732765264064/Put/seqid=0 2024-11-28T03:41:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T03:41:04,718 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:04,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-28T03:41:04,724 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T03:41:04,727 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:04,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:04,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741878_1054 (size=12595) 2024-11-28T03:41:04,757 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/e62619ff9ec34cee9b5e7d199203ac28 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e62619ff9ec34cee9b5e7d199203ac28 2024-11-28T03:41:04,770 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into e62619ff9ec34cee9b5e7d199203ac28(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:04,770 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:04,770 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=12, startTime=1732765264617; duration=0sec 2024-11-28T03:41:04,770 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:04,770 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:04,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T03:41:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:04,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/5d1f4fcc40c5457cb14f0e496d4282aa is 50, key is test_row_0/A:col10/1732765264157/Put/seqid=0 2024-11-28T03:41:04,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741879_1055 (size=14541) 2024-11-28T03:41:04,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/5d1f4fcc40c5457cb14f0e496d4282aa 2024-11-28T03:41:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T03:41:04,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/ce46a0236f71424dbf6242d2831b743a is 50, key is test_row_0/B:col10/1732765264157/Put/seqid=0 2024-11-28T03:41:04,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T03:41:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:04,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:04,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:04,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:04,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:04,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741880_1056 (size=12151) 2024-11-28T03:41:04,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/ce46a0236f71424dbf6242d2831b743a 2024-11-28T03:41:04,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765324882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765324887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765324887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765324887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765324890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:04,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/98c09ba65c4349e18d84260f8fb27364 is 50, key is test_row_0/C:col10/1732765264157/Put/seqid=0 2024-11-28T03:41:04,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741881_1057 (size=12151) 2024-11-28T03:41:04,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/98c09ba65c4349e18d84260f8fb27364 2024-11-28T03:41:04,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/5d1f4fcc40c5457cb14f0e496d4282aa as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/5d1f4fcc40c5457cb14f0e496d4282aa 2024-11-28T03:41:04,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/5d1f4fcc40c5457cb14f0e496d4282aa, entries=200, sequenceid=208, filesize=14.2 K 2024-11-28T03:41:04,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/ce46a0236f71424dbf6242d2831b743a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ce46a0236f71424dbf6242d2831b743a 2024-11-28T03:41:04,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ce46a0236f71424dbf6242d2831b743a, entries=150, sequenceid=208, filesize=11.9 K 2024-11-28T03:41:04,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/98c09ba65c4349e18d84260f8fb27364 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/98c09ba65c4349e18d84260f8fb27364 2024-11-28T03:41:04,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/98c09ba65c4349e18d84260f8fb27364, entries=150, sequenceid=208, filesize=11.9 K 2024-11-28T03:41:04,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for b7a40554a92e84435d4dc19dc444ec8c in 201ms, sequenceid=208, compaction requested=false 2024-11-28T03:41:04,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:05,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:41:05,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:05,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:05,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:05,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:05,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:05,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:05,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7dc3fc4cc6574a2abdcf61eb911e9e00 is 50, key is test_row_0/A:col10/1732765264884/Put/seqid=0 2024-11-28T03:41:05,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765325027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T03:41:05,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741882_1058 (size=12151) 2024-11-28T03:41:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:05,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:05,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:05,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:05,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765325029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7dc3fc4cc6574a2abdcf61eb911e9e00 2024-11-28T03:41:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765325029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765325032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765325030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T03:41:05,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/3754df1b358d41b49258482662f600c7 is 50, key is test_row_0/B:col10/1732765264884/Put/seqid=0 2024-11-28T03:41:05,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741883_1059 (size=12151) 2024-11-28T03:41:05,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/3754df1b358d41b49258482662f600c7 2024-11-28T03:41:05,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/3b347fe6dd5a4a12a9e3d20b58bb6cd0 is 50, key is test_row_0/C:col10/1732765264884/Put/seqid=0 2024-11-28T03:41:05,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741884_1060 (size=12151) 2024-11-28T03:41:05,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/3b347fe6dd5a4a12a9e3d20b58bb6cd0 2024-11-28T03:41:05,105 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/7552475f542a4a1c95312412280aaba7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7552475f542a4a1c95312412280aaba7 2024-11-28T03:41:05,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7dc3fc4cc6574a2abdcf61eb911e9e00 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7dc3fc4cc6574a2abdcf61eb911e9e00 2024-11-28T03:41:05,113 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into 7552475f542a4a1c95312412280aaba7(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:05,113 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:05,113 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=12, startTime=1732765264617; duration=0sec 2024-11-28T03:41:05,113 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:05,113 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:05,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7dc3fc4cc6574a2abdcf61eb911e9e00, entries=150, sequenceid=228, filesize=11.9 K 2024-11-28T03:41:05,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/3754df1b358d41b49258482662f600c7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3754df1b358d41b49258482662f600c7 2024-11-28T03:41:05,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3754df1b358d41b49258482662f600c7, entries=150, sequenceid=228, filesize=11.9 K 2024-11-28T03:41:05,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/3b347fe6dd5a4a12a9e3d20b58bb6cd0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3b347fe6dd5a4a12a9e3d20b58bb6cd0 2024-11-28T03:41:05,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765325137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3b347fe6dd5a4a12a9e3d20b58bb6cd0, entries=150, sequenceid=228, filesize=11.9 K 2024-11-28T03:41:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b7a40554a92e84435d4dc19dc444ec8c in 128ms, sequenceid=228, compaction requested=true 2024-11-28T03:41:05,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:05,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:05,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:05,141 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:05,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:05,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:05,141 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:05,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:05,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:05,142 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:05,142 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:05,142 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:05,142 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:05,142 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:05,142 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:05,142 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/2d8f849e807241c3b37b51ec6710ea87, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/5d1f4fcc40c5457cb14f0e496d4282aa, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7dc3fc4cc6574a2abdcf61eb911e9e00] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=38.4 K 2024-11-28T03:41:05,142 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7552475f542a4a1c95312412280aaba7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ce46a0236f71424dbf6242d2831b743a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3754df1b358d41b49258482662f600c7] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=36.0 K 2024-11-28T03:41:05,143 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d8f849e807241c3b37b51ec6710ea87, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732765263721 2024-11-28T03:41:05,143 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7552475f542a4a1c95312412280aaba7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732765263721 2024-11-28T03:41:05,144 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ce46a0236f71424dbf6242d2831b743a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732765264146 2024-11-28T03:41:05,145 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d1f4fcc40c5457cb14f0e496d4282aa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732765264140 2024-11-28T03:41:05,145 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3754df1b358d41b49258482662f600c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732765264881 2024-11-28T03:41:05,146 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dc3fc4cc6574a2abdcf61eb911e9e00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732765264881 2024-11-28T03:41:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:05,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T03:41:05,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:05,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:05,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:05,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:05,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:05,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:05,166 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:05,167 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/11e1b1fb1cce4272a9eef321193560b9 is 50, key is test_row_0/B:col10/1732765264884/Put/seqid=0 2024-11-28T03:41:05,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T03:41:05,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:05,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:05,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:05,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:05,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/8c1cce45660840b499495bb1e1de7f7e is 50, key is test_row_0/A:col10/1732765265154/Put/seqid=0 2024-11-28T03:41:05,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:05,186 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:05,214 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/b4a7cb714fd642e6828832808e015a18 is 50, key is test_row_0/A:col10/1732765264884/Put/seqid=0 2024-11-28T03:41:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:05,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741885_1061 (size=12697) 2024-11-28T03:41:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,232 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/11e1b1fb1cce4272a9eef321193560b9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/11e1b1fb1cce4272a9eef321193560b9 2024-11-28T03:41:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765325223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741886_1062 (size=12697) 2024-11-28T03:41:05,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765325232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765325232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,249 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into 11e1b1fb1cce4272a9eef321193560b9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765325234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,249 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,252 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=13, startTime=1732765265141; duration=0sec 2024-11-28T03:41:05,252 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:05,252 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:05,252 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:05,255 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/b4a7cb714fd642e6828832808e015a18 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b4a7cb714fd642e6828832808e015a18 2024-11-28T03:41:05,256 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:05,256 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:05,256 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:05,256 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e62619ff9ec34cee9b5e7d199203ac28, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/98c09ba65c4349e18d84260f8fb27364, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3b347fe6dd5a4a12a9e3d20b58bb6cd0] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=36.0 K 2024-11-28T03:41:05,257 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e62619ff9ec34cee9b5e7d199203ac28, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732765263721 2024-11-28T03:41:05,258 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c09ba65c4349e18d84260f8fb27364, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732765264146 2024-11-28T03:41:05,258 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b347fe6dd5a4a12a9e3d20b58bb6cd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732765264881 2024-11-28T03:41:05,267 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into b4a7cb714fd642e6828832808e015a18(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:05,267 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:05,267 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=13, startTime=1732765265140; duration=0sec 2024-11-28T03:41:05,267 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:05,267 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:05,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741887_1063 (size=14541) 2024-11-28T03:41:05,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/8c1cce45660840b499495bb1e1de7f7e 2024-11-28T03:41:05,279 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:05,280 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/cc02c0c1390d400498a13cc493154cf5 is 50, key is test_row_0/C:col10/1732765264884/Put/seqid=0 2024-11-28T03:41:05,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/fe7cd041ab4e4616a466d71d06a6d537 is 50, key is test_row_0/B:col10/1732765265154/Put/seqid=0 2024-11-28T03:41:05,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741888_1064 (size=12697) 2024-11-28T03:41:05,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741889_1065 (size=12151) 2024-11-28T03:41:05,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/fe7cd041ab4e4616a466d71d06a6d537 2024-11-28T03:41:05,319 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/cc02c0c1390d400498a13cc493154cf5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/cc02c0c1390d400498a13cc493154cf5 2024-11-28T03:41:05,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/7bb4cea39b37444f979b62fe0f3c9251 is 50, key is test_row_0/C:col10/1732765265154/Put/seqid=0 2024-11-28T03:41:05,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765325337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,344 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into cc02c0c1390d400498a13cc493154cf5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:05,344 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:05,344 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=13, startTime=1732765265141; duration=0sec 2024-11-28T03:41:05,345 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:05,345 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:05,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741890_1066 (size=12151) 2024-11-28T03:41:05,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/7bb4cea39b37444f979b62fe0f3c9251 2024-11-28T03:41:05,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765325347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T03:41:05,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/8c1cce45660840b499495bb1e1de7f7e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8c1cce45660840b499495bb1e1de7f7e 2024-11-28T03:41:05,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765325357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765325358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765325358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8c1cce45660840b499495bb1e1de7f7e, entries=200, sequenceid=245, filesize=14.2 K 2024-11-28T03:41:05,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/fe7cd041ab4e4616a466d71d06a6d537 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fe7cd041ab4e4616a466d71d06a6d537 2024-11-28T03:41:05,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T03:41:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:05,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:05,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fe7cd041ab4e4616a466d71d06a6d537, entries=150, sequenceid=245, filesize=11.9 K 2024-11-28T03:41:05,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/7bb4cea39b37444f979b62fe0f3c9251 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7bb4cea39b37444f979b62fe0f3c9251 2024-11-28T03:41:05,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7bb4cea39b37444f979b62fe0f3c9251, entries=150, sequenceid=245, filesize=11.9 K 2024-11-28T03:41:05,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for b7a40554a92e84435d4dc19dc444ec8c in 248ms, sequenceid=245, compaction requested=false 2024-11-28T03:41:05,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:05,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T03:41:05,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:05,528 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:41:05,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:05,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:05,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:05,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:05,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:05,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:05,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/b6744f3a18e84a9b94a82b013f5cfc5c is 50, key is test_row_0/A:col10/1732765265229/Put/seqid=0 2024-11-28T03:41:05,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741891_1067 (size=12251) 2024-11-28T03:41:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:05,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:05,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765325620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765325621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765325628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765325632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765325666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765325736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765325736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765325747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765325747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T03:41:05,945 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/b6744f3a18e84a9b94a82b013f5cfc5c 2024-11-28T03:41:05,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765325947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765325948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/f2ec049816a245bd8e2953169222c0b3 is 50, key is test_row_0/B:col10/1732765265229/Put/seqid=0 2024-11-28T03:41:05,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765325956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765325957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:05,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741892_1068 (size=12251) 2024-11-28T03:41:05,968 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/f2ec049816a245bd8e2953169222c0b3 2024-11-28T03:41:05,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/670dd8e8012b43bcac20d1b666077be8 is 50, key is test_row_0/C:col10/1732765265229/Put/seqid=0 2024-11-28T03:41:05,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741893_1069 (size=12251) 2024-11-28T03:41:06,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765326176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765326256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765326256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765326267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765326267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,384 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/670dd8e8012b43bcac20d1b666077be8 2024-11-28T03:41:06,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/b6744f3a18e84a9b94a82b013f5cfc5c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b6744f3a18e84a9b94a82b013f5cfc5c 2024-11-28T03:41:06,398 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b6744f3a18e84a9b94a82b013f5cfc5c, entries=150, sequenceid=266, filesize=12.0 K 2024-11-28T03:41:06,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/f2ec049816a245bd8e2953169222c0b3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/f2ec049816a245bd8e2953169222c0b3 2024-11-28T03:41:06,405 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/f2ec049816a245bd8e2953169222c0b3, entries=150, sequenceid=266, filesize=12.0 K 2024-11-28T03:41:06,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/670dd8e8012b43bcac20d1b666077be8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/670dd8e8012b43bcac20d1b666077be8 2024-11-28T03:41:06,412 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/670dd8e8012b43bcac20d1b666077be8, entries=150, sequenceid=266, filesize=12.0 K 2024-11-28T03:41:06,413 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b7a40554a92e84435d4dc19dc444ec8c in 886ms, sequenceid=266, compaction requested=true 2024-11-28T03:41:06,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:06,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:06,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-28T03:41:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-28T03:41:06,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-28T03:41:06,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6880 sec 2024-11-28T03:41:06,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7010 sec 2024-11-28T03:41:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:06,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T03:41:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:06,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7853925f2ec64e169729c439cc50750d is 50, key is test_row_0/A:col10/1732765266778/Put/seqid=0 2024-11-28T03:41:06,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741894_1070 (size=12301) 2024-11-28T03:41:06,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765326805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765326806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765326807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765326808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7853925f2ec64e169729c439cc50750d 2024-11-28T03:41:06,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/9bd6b939c7994b23be6fc6fb0957a623 is 50, key is test_row_0/B:col10/1732765266778/Put/seqid=0 2024-11-28T03:41:06,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741895_1071 (size=12301) 2024-11-28T03:41:06,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/9bd6b939c7994b23be6fc6fb0957a623 2024-11-28T03:41:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T03:41:06,877 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:06,877 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-28T03:41:06,881 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T03:41:06,882 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:06,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:06,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/d2cc516f8f5d4da99c6f51d65eae3dba is 50, key is test_row_0/C:col10/1732765266778/Put/seqid=0 2024-11-28T03:41:06,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741896_1072 (size=12301) 2024-11-28T03:41:06,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765326927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765326927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765326927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765326927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T03:41:07,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T03:41:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765327137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765327137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765327137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765327146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765327187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T03:41:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T03:41:07,200 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in b7a40554a92e84435d4dc19dc444ec8c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:41:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/d2cc516f8f5d4da99c6f51d65eae3dba 2024-11-28T03:41:07,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/7853925f2ec64e169729c439cc50750d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7853925f2ec64e169729c439cc50750d 2024-11-28T03:41:07,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7853925f2ec64e169729c439cc50750d, entries=150, sequenceid=282, filesize=12.0 K 2024-11-28T03:41:07,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/9bd6b939c7994b23be6fc6fb0957a623 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/9bd6b939c7994b23be6fc6fb0957a623 2024-11-28T03:41:07,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/9bd6b939c7994b23be6fc6fb0957a623, entries=150, sequenceid=282, filesize=12.0 K 2024-11-28T03:41:07,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/d2cc516f8f5d4da99c6f51d65eae3dba as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d2cc516f8f5d4da99c6f51d65eae3dba 2024-11-28T03:41:07,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T03:41:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d2cc516f8f5d4da99c6f51d65eae3dba, entries=150, sequenceid=282, filesize=12.0 K 2024-11-28T03:41:07,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b7a40554a92e84435d4dc19dc444ec8c in 581ms, sequenceid=282, compaction requested=true 2024-11-28T03:41:07,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:07,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:07,361 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:07,361 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:07,364 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51790 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:07,364 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:07,364 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,364 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b4a7cb714fd642e6828832808e015a18, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8c1cce45660840b499495bb1e1de7f7e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b6744f3a18e84a9b94a82b013f5cfc5c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7853925f2ec64e169729c439cc50750d] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=50.6 K 2024-11-28T03:41:07,365 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:07,365 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:07,365 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,365 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/11e1b1fb1cce4272a9eef321193560b9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fe7cd041ab4e4616a466d71d06a6d537, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/f2ec049816a245bd8e2953169222c0b3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/9bd6b939c7994b23be6fc6fb0957a623] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=48.2 K 2024-11-28T03:41:07,365 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4a7cb714fd642e6828832808e015a18, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732765264881 2024-11-28T03:41:07,366 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c1cce45660840b499495bb1e1de7f7e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732765265030 2024-11-28T03:41:07,366 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e1b1fb1cce4272a9eef321193560b9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732765264881 2024-11-28T03:41:07,366 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6744f3a18e84a9b94a82b013f5cfc5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732765265181 2024-11-28T03:41:07,366 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting fe7cd041ab4e4616a466d71d06a6d537, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732765265030 2024-11-28T03:41:07,367 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting f2ec049816a245bd8e2953169222c0b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732765265181 2024-11-28T03:41:07,367 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7853925f2ec64e169729c439cc50750d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732765265594 2024-11-28T03:41:07,368 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bd6b939c7994b23be6fc6fb0957a623, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732765265594 2024-11-28T03:41:07,385 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:07,385 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/c4f9bebb2302433ebf4ac02d73288287 is 50, key is test_row_0/A:col10/1732765266778/Put/seqid=0 2024-11-28T03:41:07,393 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:07,394 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/6916a5c6863946acab4f19abdf19f972 is 50, key is test_row_0/B:col10/1732765266778/Put/seqid=0 2024-11-28T03:41:07,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741897_1073 (size=12983) 2024-11-28T03:41:07,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741898_1074 (size=12983) 2024-11-28T03:41:07,415 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/6916a5c6863946acab4f19abdf19f972 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6916a5c6863946acab4f19abdf19f972 2024-11-28T03:41:07,422 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into 6916a5c6863946acab4f19abdf19f972(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:07,422 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:07,422 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=12, startTime=1732765267361; duration=0sec 2024-11-28T03:41:07,422 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:07,423 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:07,423 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:07,428 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:07,428 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:07,428 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,428 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/cc02c0c1390d400498a13cc493154cf5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7bb4cea39b37444f979b62fe0f3c9251, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/670dd8e8012b43bcac20d1b666077be8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d2cc516f8f5d4da99c6f51d65eae3dba] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=48.2 K 2024-11-28T03:41:07,430 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting cc02c0c1390d400498a13cc493154cf5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732765264881 2024-11-28T03:41:07,430 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bb4cea39b37444f979b62fe0f3c9251, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732765265030 2024-11-28T03:41:07,430 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 670dd8e8012b43bcac20d1b666077be8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732765265181 2024-11-28T03:41:07,431 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d2cc516f8f5d4da99c6f51d65eae3dba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732765265594 2024-11-28T03:41:07,444 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:07,444 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/82ffe6d05521429aa7997893bee483a7 is 50, key is test_row_0/C:col10/1732765266778/Put/seqid=0 2024-11-28T03:41:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:07,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T03:41:07,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:07,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:07,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:07,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:07,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:07,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:07,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/d063dfdf45df489090902e5e0ad4a920 is 50, key is test_row_0/A:col10/1732765266806/Put/seqid=0 2024-11-28T03:41:07,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741899_1075 (size=12983) 2024-11-28T03:41:07,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765327478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765327478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765327480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765327480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T03:41:07,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:07,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T03:41:07,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741900_1076 (size=14741) 2024-11-28T03:41:07,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/d063dfdf45df489090902e5e0ad4a920 2024-11-28T03:41:07,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/809abbc8df784c3db58b109293b5df93 is 50, key is test_row_0/B:col10/1732765266806/Put/seqid=0 2024-11-28T03:41:07,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741901_1077 (size=12301) 2024-11-28T03:41:07,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765327597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765327597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765327597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765327597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T03:41:07,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765327806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,807 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/c4f9bebb2302433ebf4ac02d73288287 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/c4f9bebb2302433ebf4ac02d73288287 2024-11-28T03:41:07,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765327807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T03:41:07,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:07,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,815 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into c4f9bebb2302433ebf4ac02d73288287(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:07,815 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:07,815 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=12, startTime=1732765267360; duration=0sec 2024-11-28T03:41:07,815 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:07,816 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:07,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765327817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765327818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:07,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:07,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:07,875 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/82ffe6d05521429aa7997893bee483a7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/82ffe6d05521429aa7997893bee483a7 2024-11-28T03:41:07,887 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into 82ffe6d05521429aa7997893bee483a7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:07,887 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:07,887 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=12, startTime=1732765267361; duration=0sec 2024-11-28T03:41:07,887 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:07,887 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:07,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/809abbc8df784c3db58b109293b5df93 2024-11-28T03:41:07,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/9099f69f5e214e0ca5c07af7465de646 is 50, key is test_row_0/C:col10/1732765266806/Put/seqid=0 2024-11-28T03:41:07,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T03:41:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:07,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:07,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741902_1078 (size=12301) 2024-11-28T03:41:07,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/9099f69f5e214e0ca5c07af7465de646 2024-11-28T03:41:07,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/d063dfdf45df489090902e5e0ad4a920 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/d063dfdf45df489090902e5e0ad4a920 2024-11-28T03:41:07,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/d063dfdf45df489090902e5e0ad4a920, entries=200, sequenceid=300, filesize=14.4 K 2024-11-28T03:41:07,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/809abbc8df784c3db58b109293b5df93 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/809abbc8df784c3db58b109293b5df93 2024-11-28T03:41:08,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/809abbc8df784c3db58b109293b5df93, entries=150, sequenceid=300, filesize=12.0 K 2024-11-28T03:41:08,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/9099f69f5e214e0ca5c07af7465de646 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/9099f69f5e214e0ca5c07af7465de646 2024-11-28T03:41:08,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/9099f69f5e214e0ca5c07af7465de646, entries=150, sequenceid=300, filesize=12.0 K 2024-11-28T03:41:08,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b7a40554a92e84435d4dc19dc444ec8c in 564ms, sequenceid=300, compaction requested=false 2024-11-28T03:41:08,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T03:41:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:08,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T03:41:08,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:08,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:08,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:08,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:08,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:08,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:08,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T03:41:08,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:08,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:08,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:08,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:08,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/4abab88d47fa4ff2ad57edc2e8b6ba6b is 50, key is test_row_0/A:col10/1732765268119/Put/seqid=0 2024-11-28T03:41:08,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741903_1079 (size=14741) 2024-11-28T03:41:08,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/4abab88d47fa4ff2ad57edc2e8b6ba6b 2024-11-28T03:41:08,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765328167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/2f4e5954744e41e6a23666faf8565488 is 50, key is test_row_0/B:col10/1732765268119/Put/seqid=0 2024-11-28T03:41:08,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765328172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765328172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765328174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741904_1080 (size=12301) 2024-11-28T03:41:08,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/2f4e5954744e41e6a23666faf8565488 2024-11-28T03:41:08,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/3d9fb110393d4cf59684d5fb94bfcee1 is 50, key is test_row_0/C:col10/1732765268119/Put/seqid=0 2024-11-28T03:41:08,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741905_1081 (size=12301) 2024-11-28T03:41:08,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/3d9fb110393d4cf59684d5fb94bfcee1 2024-11-28T03:41:08,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/4abab88d47fa4ff2ad57edc2e8b6ba6b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4abab88d47fa4ff2ad57edc2e8b6ba6b 2024-11-28T03:41:08,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4abab88d47fa4ff2ad57edc2e8b6ba6b, entries=200, sequenceid=319, filesize=14.4 K 2024-11-28T03:41:08,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/2f4e5954744e41e6a23666faf8565488 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/2f4e5954744e41e6a23666faf8565488 2024-11-28T03:41:08,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T03:41:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:08,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:08,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765328288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765328288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/2f4e5954744e41e6a23666faf8565488, entries=150, sequenceid=319, filesize=12.0 K 2024-11-28T03:41:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765328288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/3d9fb110393d4cf59684d5fb94bfcee1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3d9fb110393d4cf59684d5fb94bfcee1 2024-11-28T03:41:08,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765328297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3d9fb110393d4cf59684d5fb94bfcee1, entries=150, sequenceid=319, filesize=12.0 K 2024-11-28T03:41:08,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for b7a40554a92e84435d4dc19dc444ec8c in 187ms, sequenceid=319, compaction requested=true 2024-11-28T03:41:08,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:08,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:08,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:08,307 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:08,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:08,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:08,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:08,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:08,308 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:08,309 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:08,309 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:08,310 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:08,310 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/c4f9bebb2302433ebf4ac02d73288287, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/d063dfdf45df489090902e5e0ad4a920, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4abab88d47fa4ff2ad57edc2e8b6ba6b] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=41.5 K 2024-11-28T03:41:08,310 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:08,310 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:08,311 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:08,311 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6916a5c6863946acab4f19abdf19f972, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/809abbc8df784c3db58b109293b5df93, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/2f4e5954744e41e6a23666faf8565488] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=36.7 K 2024-11-28T03:41:08,311 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4f9bebb2302433ebf4ac02d73288287, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732765265594 2024-11-28T03:41:08,312 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6916a5c6863946acab4f19abdf19f972, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732765265594 2024-11-28T03:41:08,312 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d063dfdf45df489090902e5e0ad4a920, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732765266792 2024-11-28T03:41:08,312 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 809abbc8df784c3db58b109293b5df93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732765266792 2024-11-28T03:41:08,313 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4abab88d47fa4ff2ad57edc2e8b6ba6b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732765267468 2024-11-28T03:41:08,314 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4e5954744e41e6a23666faf8565488, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732765267468 2024-11-28T03:41:08,340 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:08,340 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/61cf5d93b4834d0499aa5b68e9ee85d3 is 50, key is test_row_0/A:col10/1732765268119/Put/seqid=0 2024-11-28T03:41:08,343 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:08,344 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/ecbd7f23ede64ab2aa387fa4c832c4a1 is 50, key is test_row_0/B:col10/1732765268119/Put/seqid=0 2024-11-28T03:41:08,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741906_1082 (size=13085) 2024-11-28T03:41:08,388 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/61cf5d93b4834d0499aa5b68e9ee85d3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/61cf5d93b4834d0499aa5b68e9ee85d3 2024-11-28T03:41:08,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741907_1083 (size=13085) 2024-11-28T03:41:08,398 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into 61cf5d93b4834d0499aa5b68e9ee85d3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:08,398 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:08,398 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=13, startTime=1732765268307; duration=0sec 2024-11-28T03:41:08,398 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:08,398 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:08,398 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:08,399 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:08,400 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:08,400 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:08,400 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/82ffe6d05521429aa7997893bee483a7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/9099f69f5e214e0ca5c07af7465de646, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3d9fb110393d4cf59684d5fb94bfcee1] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=36.7 K 2024-11-28T03:41:08,400 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82ffe6d05521429aa7997893bee483a7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732765265594 2024-11-28T03:41:08,401 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9099f69f5e214e0ca5c07af7465de646, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732765266792 2024-11-28T03:41:08,401 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d9fb110393d4cf59684d5fb94bfcee1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732765267468 2024-11-28T03:41:08,433 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:08,434 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/e66090b5f6224fe3ba9e96e7ec5d33d1 is 50, key is test_row_0/C:col10/1732765268119/Put/seqid=0 2024-11-28T03:41:08,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T03:41:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:08,440 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T03:41:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:08,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/21e330783b724a5a83b36a9275fe31bf is 50, key is test_row_0/A:col10/1732765268170/Put/seqid=0 2024-11-28T03:41:08,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741908_1084 (size=13085) 2024-11-28T03:41:08,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741909_1085 (size=12301) 2024-11-28T03:41:08,461 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/21e330783b724a5a83b36a9275fe31bf 2024-11-28T03:41:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/d7b053c597454a0f8634f103b4b16169 is 50, key is test_row_0/B:col10/1732765268170/Put/seqid=0 2024-11-28T03:41:08,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:08,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741910_1086 (size=12301) 2024-11-28T03:41:08,551 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/d7b053c597454a0f8634f103b4b16169 2024-11-28T03:41:08,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/541b6bd1740249ec835d6f0eca828dd1 is 50, key is test_row_0/C:col10/1732765268170/Put/seqid=0 2024-11-28T03:41:08,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741911_1087 (size=12301) 2024-11-28T03:41:08,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765328538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765328613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765328613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765328613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765328727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765328727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765328727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765328728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,802 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/ecbd7f23ede64ab2aa387fa4c832c4a1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ecbd7f23ede64ab2aa387fa4c832c4a1 2024-11-28T03:41:08,817 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into ecbd7f23ede64ab2aa387fa4c832c4a1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:08,817 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:08,817 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=13, startTime=1732765268307; duration=0sec 2024-11-28T03:41:08,818 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:08,818 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:08,866 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/e66090b5f6224fe3ba9e96e7ec5d33d1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e66090b5f6224fe3ba9e96e7ec5d33d1 2024-11-28T03:41:08,880 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into e66090b5f6224fe3ba9e96e7ec5d33d1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:08,880 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:08,880 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=13, startTime=1732765268307; duration=0sec 2024-11-28T03:41:08,881 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:08,881 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:08,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765328937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765328937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765328937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765328937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:08,973 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/541b6bd1740249ec835d6f0eca828dd1 2024-11-28T03:41:08,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/21e330783b724a5a83b36a9275fe31bf as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/21e330783b724a5a83b36a9275fe31bf 2024-11-28T03:41:08,992 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/21e330783b724a5a83b36a9275fe31bf, entries=150, sequenceid=336, filesize=12.0 K 2024-11-28T03:41:08,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/d7b053c597454a0f8634f103b4b16169 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/d7b053c597454a0f8634f103b4b16169 2024-11-28T03:41:09,005 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/d7b053c597454a0f8634f103b4b16169, entries=150, sequenceid=336, filesize=12.0 K 2024-11-28T03:41:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/541b6bd1740249ec835d6f0eca828dd1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/541b6bd1740249ec835d6f0eca828dd1 2024-11-28T03:41:09,018 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/541b6bd1740249ec835d6f0eca828dd1, entries=150, sequenceid=336, filesize=12.0 K 2024-11-28T03:41:09,020 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for b7a40554a92e84435d4dc19dc444ec8c in 580ms, sequenceid=336, compaction requested=false 2024-11-28T03:41:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-28T03:41:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-28T03:41:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T03:41:09,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-28T03:41:09,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1410 sec 2024-11-28T03:41:09,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.1530 sec 2024-11-28T03:41:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:09,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:41:09,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:09,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:09,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:09,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:09,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:09,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:09,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/626c1653490b464cb2c322c2044b5563 is 50, key is test_row_0/A:col10/1732765268546/Put/seqid=0 2024-11-28T03:41:09,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765329259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765329260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765329263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765329265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765329266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741912_1088 (size=12301) 2024-11-28T03:41:09,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/626c1653490b464cb2c322c2044b5563 2024-11-28T03:41:09,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/48830e660a2f4095a77a69ab05193e55 is 50, key is test_row_0/B:col10/1732765268546/Put/seqid=0 2024-11-28T03:41:09,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741913_1089 (size=12301) 2024-11-28T03:41:09,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/48830e660a2f4095a77a69ab05193e55 2024-11-28T03:41:09,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/faf50454bd154698ac727993fc3622b7 is 50, key is test_row_0/C:col10/1732765268546/Put/seqid=0 2024-11-28T03:41:09,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741914_1090 (size=12301) 2024-11-28T03:41:09,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765329377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765329377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765329377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765329378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765329589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765329589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765329589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765329589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/faf50454bd154698ac727993fc3622b7 2024-11-28T03:41:09,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/626c1653490b464cb2c322c2044b5563 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/626c1653490b464cb2c322c2044b5563 2024-11-28T03:41:09,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765329777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/626c1653490b464cb2c322c2044b5563, entries=150, sequenceid=357, filesize=12.0 K 2024-11-28T03:41:09,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/48830e660a2f4095a77a69ab05193e55 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/48830e660a2f4095a77a69ab05193e55 2024-11-28T03:41:09,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/48830e660a2f4095a77a69ab05193e55, entries=150, sequenceid=357, filesize=12.0 K 2024-11-28T03:41:09,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/faf50454bd154698ac727993fc3622b7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/faf50454bd154698ac727993fc3622b7 2024-11-28T03:41:09,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/faf50454bd154698ac727993fc3622b7, entries=150, sequenceid=357, filesize=12.0 K 2024-11-28T03:41:09,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b7a40554a92e84435d4dc19dc444ec8c in 596ms, sequenceid=357, compaction requested=true 2024-11-28T03:41:09,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:09,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:09,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:09,817 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:09,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:09,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:09,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:09,817 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:09,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:09,818 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:09,818 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:09,818 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:09,818 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:09,818 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:09,818 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/61cf5d93b4834d0499aa5b68e9ee85d3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/21e330783b724a5a83b36a9275fe31bf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/626c1653490b464cb2c322c2044b5563] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=36.8 K 2024-11-28T03:41:09,818 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:09,819 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ecbd7f23ede64ab2aa387fa4c832c4a1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/d7b053c597454a0f8634f103b4b16169, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/48830e660a2f4095a77a69ab05193e55] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=36.8 K 2024-11-28T03:41:09,820 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 61cf5d93b4834d0499aa5b68e9ee85d3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732765267468 2024-11-28T03:41:09,820 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecbd7f23ede64ab2aa387fa4c832c4a1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732765267468 2024-11-28T03:41:09,820 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 21e330783b724a5a83b36a9275fe31bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732765268157 2024-11-28T03:41:09,820 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7b053c597454a0f8634f103b4b16169, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732765268157 2024-11-28T03:41:09,821 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 626c1653490b464cb2c322c2044b5563, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732765268541 2024-11-28T03:41:09,821 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48830e660a2f4095a77a69ab05193e55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732765268541 2024-11-28T03:41:09,839 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:09,842 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/bec7a1e131e74f6a8f94187c02ec5518 is 50, key is test_row_0/B:col10/1732765268546/Put/seqid=0 2024-11-28T03:41:09,843 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:09,844 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/4282d0aad6dc4c9c9c2f3e6765eeee4a is 50, key is test_row_0/A:col10/1732765268546/Put/seqid=0 2024-11-28T03:41:09,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741916_1092 (size=13187) 2024-11-28T03:41:09,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741915_1091 (size=13187) 2024-11-28T03:41:09,866 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/bec7a1e131e74f6a8f94187c02ec5518 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/bec7a1e131e74f6a8f94187c02ec5518 2024-11-28T03:41:09,891 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into bec7a1e131e74f6a8f94187c02ec5518(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:09,891 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:09,891 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=13, startTime=1732765269817; duration=0sec 2024-11-28T03:41:09,891 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:09,891 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:09,891 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:09,895 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:09,895 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:09,895 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:09,895 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e66090b5f6224fe3ba9e96e7ec5d33d1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/541b6bd1740249ec835d6f0eca828dd1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/faf50454bd154698ac727993fc3622b7] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=36.8 K 2024-11-28T03:41:09,896 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e66090b5f6224fe3ba9e96e7ec5d33d1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732765267468 2024-11-28T03:41:09,896 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 541b6bd1740249ec835d6f0eca828dd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732765268157 2024-11-28T03:41:09,897 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting faf50454bd154698ac727993fc3622b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732765268541 2024-11-28T03:41:09,909 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:09,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:41:09,910 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/e186ecd23d034843a3bbae371fa46d92 is 50, key is test_row_0/C:col10/1732765268546/Put/seqid=0 2024-11-28T03:41:09,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:09,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:09,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:09,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:09,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:09,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:09,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/300f97c3fc014ced88de62e1087f4934 is 50, key is test_row_0/A:col10/1732765269908/Put/seqid=0 2024-11-28T03:41:09,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741917_1093 (size=13187) 2024-11-28T03:41:09,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741918_1094 (size=12301) 2024-11-28T03:41:09,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765329980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765329982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765329985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765329987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765330096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765330096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765330097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765330097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,275 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/4282d0aad6dc4c9c9c2f3e6765eeee4a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4282d0aad6dc4c9c9c2f3e6765eeee4a 2024-11-28T03:41:10,289 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into 4282d0aad6dc4c9c9c2f3e6765eeee4a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:10,289 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:10,289 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=13, startTime=1732765269816; duration=0sec 2024-11-28T03:41:10,289 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:10,289 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:10,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765330307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765330316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765330318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765330316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,334 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/e186ecd23d034843a3bbae371fa46d92 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e186ecd23d034843a3bbae371fa46d92 2024-11-28T03:41:10,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/300f97c3fc014ced88de62e1087f4934 2024-11-28T03:41:10,351 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into e186ecd23d034843a3bbae371fa46d92(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:10,351 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:10,351 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=13, startTime=1732765269817; duration=0sec 2024-11-28T03:41:10,352 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:10,352 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:10,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/3cb103e0b3fa4255873cda52361332f3 is 50, key is test_row_0/B:col10/1732765269908/Put/seqid=0 2024-11-28T03:41:10,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741919_1095 (size=12301) 2024-11-28T03:41:10,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765330616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765330626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765330626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765330636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765330786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:10,792 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in b7a40554a92e84435d4dc19dc444ec8c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:41:10,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:10,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:10,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/3cb103e0b3fa4255873cda52361332f3 2024-11-28T03:41:10,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/45c4520471324514a7ffd7768d23258b is 50, key is test_row_0/C:col10/1732765269908/Put/seqid=0 2024-11-28T03:41:10,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741920_1096 (size=12301) 2024-11-28T03:41:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T03:41:11,047 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:11,048 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-28T03:41:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T03:41:11,051 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:11,052 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:11,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:11,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765331131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:11,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765331138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:11,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765331150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:11,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765331150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T03:41:11,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T03:41:11,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:11,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:11,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:11,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:11,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:11,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/45c4520471324514a7ffd7768d23258b 2024-11-28T03:41:11,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/300f97c3fc014ced88de62e1087f4934 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/300f97c3fc014ced88de62e1087f4934 2024-11-28T03:41:11,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/300f97c3fc014ced88de62e1087f4934, entries=150, sequenceid=373, filesize=12.0 K 2024-11-28T03:41:11,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/3cb103e0b3fa4255873cda52361332f3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cb103e0b3fa4255873cda52361332f3 2024-11-28T03:41:11,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cb103e0b3fa4255873cda52361332f3, entries=150, sequenceid=373, filesize=12.0 K 2024-11-28T03:41:11,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/45c4520471324514a7ffd7768d23258b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/45c4520471324514a7ffd7768d23258b 2024-11-28T03:41:11,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/45c4520471324514a7ffd7768d23258b, entries=150, sequenceid=373, filesize=12.0 K 2024-11-28T03:41:11,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b7a40554a92e84435d4dc19dc444ec8c in 1410ms, sequenceid=373, compaction requested=false 2024-11-28T03:41:11,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:11,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T03:41:11,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:11,366 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T03:41:11,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:11,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:11,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:11,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:11,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:11,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T03:41:11,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/cb976ccb71cb4dbbbabed04cbfadbe3d is 50, key is test_row_0/A:col10/1732765269932/Put/seqid=0 2024-11-28T03:41:11,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741921_1097 (size=12301) 2024-11-28T03:41:11,424 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/cb976ccb71cb4dbbbabed04cbfadbe3d 2024-11-28T03:41:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/071f3cd480fb4f4baff8bb2d8f034cac is 50, key is test_row_0/B:col10/1732765269932/Put/seqid=0 2024-11-28T03:41:11,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741922_1098 (size=12301) 2024-11-28T03:41:11,474 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/071f3cd480fb4f4baff8bb2d8f034cac 2024-11-28T03:41:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/15e54d33621b47c18e4c2adf732b5232 is 50, key is test_row_0/C:col10/1732765269932/Put/seqid=0 2024-11-28T03:41:11,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741923_1099 (size=12301) 2024-11-28T03:41:11,536 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/15e54d33621b47c18e4c2adf732b5232 2024-11-28T03:41:11,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/cb976ccb71cb4dbbbabed04cbfadbe3d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cb976ccb71cb4dbbbabed04cbfadbe3d 2024-11-28T03:41:11,559 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cb976ccb71cb4dbbbabed04cbfadbe3d, entries=150, sequenceid=392, filesize=12.0 K 2024-11-28T03:41:11,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/071f3cd480fb4f4baff8bb2d8f034cac as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/071f3cd480fb4f4baff8bb2d8f034cac 2024-11-28T03:41:11,579 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/071f3cd480fb4f4baff8bb2d8f034cac, entries=150, sequenceid=392, filesize=12.0 K 2024-11-28T03:41:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/15e54d33621b47c18e4c2adf732b5232 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/15e54d33621b47c18e4c2adf732b5232 2024-11-28T03:41:11,595 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/15e54d33621b47c18e4c2adf732b5232, entries=150, sequenceid=392, filesize=12.0 K 2024-11-28T03:41:11,597 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for b7a40554a92e84435d4dc19dc444ec8c in 231ms, sequenceid=392, compaction requested=true 2024-11-28T03:41:11,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:11,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:11,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-28T03:41:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-28T03:41:11,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-28T03:41:11,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 548 msec 2024-11-28T03:41:11,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 555 msec 2024-11-28T03:41:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T03:41:11,677 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:11,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-28T03:41:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T03:41:11,681 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:11,682 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:11,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T03:41:11,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T03:41:11,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:11,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:11,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:11,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-28T03:41:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-28T03:41:11,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-28T03:41:11,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 163 msec 2024-11-28T03:41:11,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 173 msec 2024-11-28T03:41:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T03:41:11,997 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:11,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-28T03:41:12,001 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T03:41:12,002 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:12,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T03:41:12,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T03:41:12,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:12,157 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-28T03:41:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:12,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/6032b30a4f004df0905d8585f7bcf64d is 50, key is test_row_0/A:col10/1732765272154/Put/seqid=0 2024-11-28T03:41:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:12,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:12,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741924_1100 (size=9855) 2024-11-28T03:41:12,211 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/6032b30a4f004df0905d8585f7bcf64d 2024-11-28T03:41:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/a54f4cdbd99e4792837f5b461deb1452 is 50, key is test_row_0/B:col10/1732765272154/Put/seqid=0 2024-11-28T03:41:12,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741925_1101 (size=7415) 2024-11-28T03:41:12,243 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/a54f4cdbd99e4792837f5b461deb1452 2024-11-28T03:41:12,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/277ae4b675674bce9776e3a080fdfe77 is 50, key is test_row_0/C:col10/1732765272154/Put/seqid=0 2024-11-28T03:41:12,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765272175, max timestamp=1732765272212Num uniques 859; 2024-11-28T03:41:12,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:41:12,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765272175, max timestamp=1732765272231Num uniques 859; 2024-11-28T03:41:12,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:41:12,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732765272175, max timestamp=1732765272224Num uniques 859; 2024-11-28T03:41:12,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T03:41:12,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741926_1102 (size=7415) 2024-11-28T03:41:12,308 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/277ae4b675674bce9776e3a080fdfe77 2024-11-28T03:41:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T03:41:12,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/6032b30a4f004df0905d8585f7bcf64d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/6032b30a4f004df0905d8585f7bcf64d 2024-11-28T03:41:12,323 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/6032b30a4f004df0905d8585f7bcf64d, entries=100, sequenceid=398, filesize=9.6 K 2024-11-28T03:41:12,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/a54f4cdbd99e4792837f5b461deb1452 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a54f4cdbd99e4792837f5b461deb1452 2024-11-28T03:41:12,334 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a54f4cdbd99e4792837f5b461deb1452, entries=50, sequenceid=398, filesize=7.2 K 2024-11-28T03:41:12,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/277ae4b675674bce9776e3a080fdfe77 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/277ae4b675674bce9776e3a080fdfe77 2024-11-28T03:41:12,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765272316, max timestamp=1732765272331Num uniques 1018; 2024-11-28T03:41:12,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T03:41:12,346 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/277ae4b675674bce9776e3a080fdfe77, entries=50, sequenceid=398, filesize=7.2 K 2024-11-28T03:41:12,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765272316, max timestamp=1732765272331Num uniques 1009; 2024-11-28T03:41:12,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-28T03:41:12,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765272316, max timestamp=1732765272331Num uniques 1018; 2024-11-28T03:41:12,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T03:41:12,356 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=46.96 KB/48090 for b7a40554a92e84435d4dc19dc444ec8c in 199ms, sequenceid=398, compaction requested=true 2024-11-28T03:41:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-28T03:41:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-28T03:41:12,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-28T03:41:12,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 355 msec 2024-11-28T03:41:12,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 364 msec 2024-11-28T03:41:12,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-11-28T03:41:12,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:12,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:12,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:12,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:12,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:12,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:12,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/422aae67049d484db3a62b68a599169f is 50, key is test_row_0/A:col10/1732765272368/Put/seqid=0 2024-11-28T03:41:12,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741927_1103 (size=24505) 2024-11-28T03:41:12,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765272479, max timestamp=1732765272508Num uniques 859; 2024-11-28T03:41:12,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:41:12,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765272479, max timestamp=1732765272508Num uniques 859; 2024-11-28T03:41:12,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:41:12,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765272479, max timestamp=1732765272508Num uniques 859; 2024-11-28T03:41:12,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:41:12,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765332576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765332581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765332585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765332586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T03:41:12,627 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:12,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-28T03:41:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T03:41:12,630 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:12,631 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:12,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:12,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765332696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765332706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765332707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765332707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T03:41:12,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T03:41:12,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:12,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:12,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:12,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:12,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:12,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765332806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,814 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in b7a40554a92e84435d4dc19dc444ec8c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:41:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/422aae67049d484db3a62b68a599169f 2024-11-28T03:41:12,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765332919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765332926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765332927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765332926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:12,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T03:41:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:12,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:12,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/dd8c05fafe3b4754aea5d19038b024f9 is 50, key is test_row_0/B:col10/1732765272368/Put/seqid=0 2024-11-28T03:41:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T03:41:12,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741928_1104 (size=12301) 2024-11-28T03:41:12,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/dd8c05fafe3b4754aea5d19038b024f9 2024-11-28T03:41:13,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/74166a4c95134f7eb68e312a4b301b74 is 50, key is test_row_0/C:col10/1732765272368/Put/seqid=0 2024-11-28T03:41:13,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741929_1105 (size=12301) 2024-11-28T03:41:13,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T03:41:13,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:13,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:13,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:13,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:13,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:13,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765333237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765333237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765333237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765333237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T03:41:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:13,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:13,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T03:41:13,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T03:41:13,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:13,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:13,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:13,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:13,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:13,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/74166a4c95134f7eb68e312a4b301b74 2024-11-28T03:41:13,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/422aae67049d484db3a62b68a599169f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/422aae67049d484db3a62b68a599169f 2024-11-28T03:41:13,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/422aae67049d484db3a62b68a599169f, entries=400, sequenceid=440, filesize=23.9 K 2024-11-28T03:41:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/dd8c05fafe3b4754aea5d19038b024f9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dd8c05fafe3b4754aea5d19038b024f9 2024-11-28T03:41:13,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dd8c05fafe3b4754aea5d19038b024f9, entries=150, sequenceid=440, filesize=12.0 K 2024-11-28T03:41:13,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/74166a4c95134f7eb68e312a4b301b74 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/74166a4c95134f7eb68e312a4b301b74 2024-11-28T03:41:13,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/74166a4c95134f7eb68e312a4b301b74, entries=150, sequenceid=440, filesize=12.0 K 2024-11-28T03:41:13,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=134.18 KB/137400 for b7a40554a92e84435d4dc19dc444ec8c in 1125ms, sequenceid=440, compaction requested=true 2024-11-28T03:41:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:13,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:13,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:13,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:13,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T03:41:13,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:13,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-28T03:41:13,496 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T03:41:13,496 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T03:41:13,499 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 72149 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T03:41:13,499 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57505 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T03:41:13,499 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:13,499 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:13,499 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:13,499 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:13,499 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4282d0aad6dc4c9c9c2f3e6765eeee4a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/300f97c3fc014ced88de62e1087f4934, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cb976ccb71cb4dbbbabed04cbfadbe3d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/6032b30a4f004df0905d8585f7bcf64d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/422aae67049d484db3a62b68a599169f] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=70.5 K 2024-11-28T03:41:13,499 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e186ecd23d034843a3bbae371fa46d92, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/45c4520471324514a7ffd7768d23258b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/15e54d33621b47c18e4c2adf732b5232, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/277ae4b675674bce9776e3a080fdfe77, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/74166a4c95134f7eb68e312a4b301b74] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=56.2 K 2024-11-28T03:41:13,500 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e186ecd23d034843a3bbae371fa46d92, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732765268541 2024-11-28T03:41:13,500 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4282d0aad6dc4c9c9c2f3e6765eeee4a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732765268541 2024-11-28T03:41:13,500 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 300f97c3fc014ced88de62e1087f4934, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732765269225 2024-11-28T03:41:13,500 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c4520471324514a7ffd7768d23258b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732765269225 2024-11-28T03:41:13,501 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting cb976ccb71cb4dbbbabed04cbfadbe3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732765269932 2024-11-28T03:41:13,501 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15e54d33621b47c18e4c2adf732b5232, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732765269932 2024-11-28T03:41:13,501 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6032b30a4f004df0905d8585f7bcf64d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732765272149 2024-11-28T03:41:13,502 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 277ae4b675674bce9776e3a080fdfe77, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732765272154 2024-11-28T03:41:13,502 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 422aae67049d484db3a62b68a599169f, keycount=400, bloomtype=ROW, size=23.9 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732765272316 2024-11-28T03:41:13,502 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74166a4c95134f7eb68e312a4b301b74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732765272331 2024-11-28T03:41:13,523 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#93 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:13,524 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/82f61dc5fef44668903e4f5a3325c50a is 50, key is test_row_0/A:col10/1732765272368/Put/seqid=0 2024-11-28T03:41:13,539 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#94 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:13,540 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/fffc7fc6dc054bea80008d3483d35366 is 50, key is test_row_0/C:col10/1732765272368/Put/seqid=0 2024-11-28T03:41:13,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741930_1106 (size=13357) 2024-11-28T03:41:13,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T03:41:13,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:13,559 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T03:41:13,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:13,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:13,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:13,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:13,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:13,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:13,560 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/82f61dc5fef44668903e4f5a3325c50a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/82f61dc5fef44668903e4f5a3325c50a 2024-11-28T03:41:13,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741931_1107 (size=13357) 2024-11-28T03:41:13,575 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into 82f61dc5fef44668903e4f5a3325c50a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:13,575 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:13,575 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=11, startTime=1732765273495; duration=0sec 2024-11-28T03:41:13,576 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:13,576 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:13,576 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T03:41:13,584 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/fffc7fc6dc054bea80008d3483d35366 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/fffc7fc6dc054bea80008d3483d35366 2024-11-28T03:41:13,585 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57505 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T03:41:13,586 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:13,586 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:13,586 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/bec7a1e131e74f6a8f94187c02ec5518, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cb103e0b3fa4255873cda52361332f3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/071f3cd480fb4f4baff8bb2d8f034cac, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a54f4cdbd99e4792837f5b461deb1452, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dd8c05fafe3b4754aea5d19038b024f9] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=56.2 K 2024-11-28T03:41:13,590 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting bec7a1e131e74f6a8f94187c02ec5518, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732765268541 2024-11-28T03:41:13,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/f2f22210de7d45338779a394fbf32c1e is 50, key is test_row_0/A:col10/1732765272580/Put/seqid=0 2024-11-28T03:41:13,592 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cb103e0b3fa4255873cda52361332f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732765269225 2024-11-28T03:41:13,593 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into fffc7fc6dc054bea80008d3483d35366(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:13,593 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:13,593 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=11, startTime=1732765273496; duration=0sec 2024-11-28T03:41:13,593 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:13,593 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:13,593 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 071f3cd480fb4f4baff8bb2d8f034cac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732765269932 2024-11-28T03:41:13,594 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting a54f4cdbd99e4792837f5b461deb1452, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732765272154 2024-11-28T03:41:13,594 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting dd8c05fafe3b4754aea5d19038b024f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732765272331 2024-11-28T03:41:13,615 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#96 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:13,616 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/3cd3f15527a1413e85a176fcdcb3968c is 50, key is test_row_0/B:col10/1732765272368/Put/seqid=0 2024-11-28T03:41:13,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741932_1108 (size=12301) 2024-11-28T03:41:13,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741933_1109 (size=13357) 2024-11-28T03:41:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T03:41:13,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765333767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765333775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765333776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765333779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765333887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765333887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765333887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:13,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765333897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,046 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/f2f22210de7d45338779a394fbf32c1e 2024-11-28T03:41:14,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/292ca14fc59f4ee38e010766d6a3f3ab is 50, key is test_row_0/B:col10/1732765272580/Put/seqid=0 2024-11-28T03:41:14,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741934_1110 (size=12301) 2024-11-28T03:41:14,096 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/292ca14fc59f4ee38e010766d6a3f3ab 2024-11-28T03:41:14,097 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/3cd3f15527a1413e85a176fcdcb3968c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cd3f15527a1413e85a176fcdcb3968c 2024-11-28T03:41:14,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765334097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765334097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765334098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765334106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/7ef58fa3b970439a974a7f6c9fdb11c0 is 50, key is test_row_0/C:col10/1732765272580/Put/seqid=0 2024-11-28T03:41:14,120 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into 3cd3f15527a1413e85a176fcdcb3968c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:14,120 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:14,120 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=11, startTime=1732765273496; duration=0sec 2024-11-28T03:41:14,121 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:14,121 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:14,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741935_1111 (size=12301) 2024-11-28T03:41:14,127 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/7ef58fa3b970439a974a7f6c9fdb11c0 2024-11-28T03:41:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/f2f22210de7d45338779a394fbf32c1e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/f2f22210de7d45338779a394fbf32c1e 2024-11-28T03:41:14,148 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/f2f22210de7d45338779a394fbf32c1e, entries=150, sequenceid=477, filesize=12.0 K 2024-11-28T03:41:14,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/292ca14fc59f4ee38e010766d6a3f3ab as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/292ca14fc59f4ee38e010766d6a3f3ab 2024-11-28T03:41:14,162 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/292ca14fc59f4ee38e010766d6a3f3ab, entries=150, sequenceid=477, filesize=12.0 K 2024-11-28T03:41:14,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/7ef58fa3b970439a974a7f6c9fdb11c0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ef58fa3b970439a974a7f6c9fdb11c0 2024-11-28T03:41:14,179 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ef58fa3b970439a974a7f6c9fdb11c0, entries=150, sequenceid=477, filesize=12.0 K 2024-11-28T03:41:14,180 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for b7a40554a92e84435d4dc19dc444ec8c in 621ms, sequenceid=477, compaction requested=false 2024-11-28T03:41:14,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:14,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:14,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-28T03:41:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-28T03:41:14,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.5570 sec 2024-11-28T03:41:14,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-28T03:41:14,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5510 sec 2024-11-28T03:41:14,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T03:41:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:14,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/cc9566afe4144b7c861f91a057ecf56a is 50, key is test_row_0/A:col10/1732765274420/Put/seqid=0 2024-11-28T03:41:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765334453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765334455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765334456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765334464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741936_1112 (size=14741) 2024-11-28T03:41:14,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/cc9566afe4144b7c861f91a057ecf56a 2024-11-28T03:41:14,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/dbb5c56c3426471d8f6022d0e48fee9b is 50, key is test_row_0/B:col10/1732765274420/Put/seqid=0 2024-11-28T03:41:14,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741937_1113 (size=12301) 2024-11-28T03:41:14,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/dbb5c56c3426471d8f6022d0e48fee9b 2024-11-28T03:41:14,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/b059235207044c1999b61ae0acc474f4 is 50, key is test_row_0/C:col10/1732765274420/Put/seqid=0 2024-11-28T03:41:14,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741938_1114 (size=12301) 2024-11-28T03:41:14,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/b059235207044c1999b61ae0acc474f4 2024-11-28T03:41:14,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/cc9566afe4144b7c861f91a057ecf56a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc9566afe4144b7c861f91a057ecf56a 2024-11-28T03:41:14,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc9566afe4144b7c861f91a057ecf56a, entries=200, sequenceid=494, filesize=14.4 K 2024-11-28T03:41:14,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/dbb5c56c3426471d8f6022d0e48fee9b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dbb5c56c3426471d8f6022d0e48fee9b 2024-11-28T03:41:14,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dbb5c56c3426471d8f6022d0e48fee9b, entries=150, sequenceid=494, filesize=12.0 K 2024-11-28T03:41:14,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765334576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765334576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765334576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765334577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/b059235207044c1999b61ae0acc474f4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b059235207044c1999b61ae0acc474f4 2024-11-28T03:41:14,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b059235207044c1999b61ae0acc474f4, entries=150, sequenceid=494, filesize=12.0 K 2024-11-28T03:41:14,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b7a40554a92e84435d4dc19dc444ec8c in 175ms, sequenceid=494, compaction requested=true 2024-11-28T03:41:14,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:14,595 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:14,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:14,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:14,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:14,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:14,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:14,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T03:41:14,598 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:14,599 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:14,599 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:14,599 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:14,599 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/82f61dc5fef44668903e4f5a3325c50a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/f2f22210de7d45338779a394fbf32c1e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc9566afe4144b7c861f91a057ecf56a] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=39.5 K 2024-11-28T03:41:14,599 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:14,600 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:14,600 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:14,600 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cd3f15527a1413e85a176fcdcb3968c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/292ca14fc59f4ee38e010766d6a3f3ab, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dbb5c56c3426471d8f6022d0e48fee9b] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=37.1 K 2024-11-28T03:41:14,600 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82f61dc5fef44668903e4f5a3325c50a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732765272331 2024-11-28T03:41:14,600 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cd3f15527a1413e85a176fcdcb3968c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732765272331 2024-11-28T03:41:14,601 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2f22210de7d45338779a394fbf32c1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732765272562 2024-11-28T03:41:14,601 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 292ca14fc59f4ee38e010766d6a3f3ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732765272562 2024-11-28T03:41:14,601 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc9566afe4144b7c861f91a057ecf56a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732765273764 2024-11-28T03:41:14,602 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting dbb5c56c3426471d8f6022d0e48fee9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732765273775 2024-11-28T03:41:14,618 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:14,619 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/da30e89360474b69981f2495c65341bc is 50, key is test_row_0/B:col10/1732765274420/Put/seqid=0 2024-11-28T03:41:14,633 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#103 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:14,634 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/a277f8eb0dc54508ba6f6adbaea8e93c is 50, key is test_row_0/A:col10/1732765274420/Put/seqid=0 2024-11-28T03:41:14,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741940_1116 (size=13459) 2024-11-28T03:41:14,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741939_1115 (size=13459) 2024-11-28T03:41:14,680 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/da30e89360474b69981f2495c65341bc as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da30e89360474b69981f2495c65341bc 2024-11-28T03:41:14,691 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into da30e89360474b69981f2495c65341bc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:14,691 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:14,691 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=13, startTime=1732765274596; duration=0sec 2024-11-28T03:41:14,691 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:14,691 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:14,691 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:14,695 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:14,696 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:14,696 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:14,696 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/fffc7fc6dc054bea80008d3483d35366, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ef58fa3b970439a974a7f6c9fdb11c0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b059235207044c1999b61ae0acc474f4] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=37.1 K 2024-11-28T03:41:14,697 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting fffc7fc6dc054bea80008d3483d35366, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732765272331 2024-11-28T03:41:14,697 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ef58fa3b970439a974a7f6c9fdb11c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732765272562 2024-11-28T03:41:14,698 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting b059235207044c1999b61ae0acc474f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732765273775 2024-11-28T03:41:14,728 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:14,729 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/7ead034adcab49abb200e226e0f33671 is 50, key is test_row_0/C:col10/1732765274420/Put/seqid=0 2024-11-28T03:41:14,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741941_1117 (size=13459) 2024-11-28T03:41:14,763 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/7ead034adcab49abb200e226e0f33671 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ead034adcab49abb200e226e0f33671 2024-11-28T03:41:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T03:41:14,777 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:14,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-28T03:41:14,782 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T03:41:14,783 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:14,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:14,787 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into 7ead034adcab49abb200e226e0f33671(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:14,787 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:14,787 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=13, startTime=1732765274596; duration=0sec 2024-11-28T03:41:14,787 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:14,787 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:14,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:41:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:14,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:14,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:14,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:14,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:14,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/88e4db5212d34b54978e3b7c7d06d881 is 50, key is test_row_0/A:col10/1732765274790/Put/seqid=0 2024-11-28T03:41:14,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765334810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765334817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741942_1118 (size=17181) 2024-11-28T03:41:14,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/88e4db5212d34b54978e3b7c7d06d881 2024-11-28T03:41:14,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765334832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765334832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/62d32d8172dc4f3a93af1665d1f2d2c4 is 50, key is test_row_0/B:col10/1732765274790/Put/seqid=0 2024-11-28T03:41:14,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741943_1119 (size=12301) 2024-11-28T03:41:14,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/62d32d8172dc4f3a93af1665d1f2d2c4 2024-11-28T03:41:14,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/b9727f0b6f3641ebae89d1956310b343 is 50, key is test_row_0/C:col10/1732765274790/Put/seqid=0 2024-11-28T03:41:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T03:41:14,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741944_1120 (size=12301) 2024-11-28T03:41:14,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/b9727f0b6f3641ebae89d1956310b343 2024-11-28T03:41:14,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/88e4db5212d34b54978e3b7c7d06d881 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/88e4db5212d34b54978e3b7c7d06d881 2024-11-28T03:41:14,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/88e4db5212d34b54978e3b7c7d06d881, entries=250, sequenceid=516, filesize=16.8 K 2024-11-28T03:41:14,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/62d32d8172dc4f3a93af1665d1f2d2c4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/62d32d8172dc4f3a93af1665d1f2d2c4 2024-11-28T03:41:14,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/62d32d8172dc4f3a93af1665d1f2d2c4, entries=150, sequenceid=516, filesize=12.0 K 2024-11-28T03:41:14,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/b9727f0b6f3641ebae89d1956310b343 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b9727f0b6f3641ebae89d1956310b343 2024-11-28T03:41:14,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b9727f0b6f3641ebae89d1956310b343, entries=150, sequenceid=516, filesize=12.0 K 2024-11-28T03:41:14,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for b7a40554a92e84435d4dc19dc444ec8c in 136ms, sequenceid=516, compaction requested=false 2024-11-28T03:41:14,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:14,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:41:14,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:14,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:14,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:14,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:14,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:14,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:14,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T03:41:14,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:14,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:14,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:14,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:14,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:14,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/e11555900c274c85b71e2333ac3b6d41 is 50, key is test_row_0/A:col10/1732765274803/Put/seqid=0 2024-11-28T03:41:14,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741945_1121 (size=14741) 2024-11-28T03:41:14,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/e11555900c274c85b71e2333ac3b6d41 2024-11-28T03:41:14,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/1eeb194d63e3434b99268848809f3453 is 50, key is test_row_0/B:col10/1732765274803/Put/seqid=0 2024-11-28T03:41:14,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765334977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765334980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765334983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:14,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765334993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741946_1122 (size=12301) 2024-11-28T03:41:15,073 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/a277f8eb0dc54508ba6f6adbaea8e93c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a277f8eb0dc54508ba6f6adbaea8e93c 2024-11-28T03:41:15,092 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into a277f8eb0dc54508ba6f6adbaea8e93c(size=13.1 K), total size for store is 29.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:15,092 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:15,092 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=13, startTime=1732765274595; duration=0sec 2024-11-28T03:41:15,093 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:15,093 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:15,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T03:41:15,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:15,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T03:41:15,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765335097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765335097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765335106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765335106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T03:41:15,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:15,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765335317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765335317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765335317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765335321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T03:41:15,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T03:41:15,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/1eeb194d63e3434b99268848809f3453 2024-11-28T03:41:15,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/c011b4f51eeb4490a843ad2a62054414 is 50, key is test_row_0/C:col10/1732765274803/Put/seqid=0 2024-11-28T03:41:15,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741947_1123 (size=12301) 2024-11-28T03:41:15,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T03:41:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765335626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765335626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765335637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765335639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:15,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:15,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T03:41:15,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:15,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/c011b4f51eeb4490a843ad2a62054414 2024-11-28T03:41:15,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/e11555900c274c85b71e2333ac3b6d41 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/e11555900c274c85b71e2333ac3b6d41 2024-11-28T03:41:15,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T03:41:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:15,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/e11555900c274c85b71e2333ac3b6d41, entries=200, sequenceid=530, filesize=14.4 K 2024-11-28T03:41:15,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/1eeb194d63e3434b99268848809f3453 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1eeb194d63e3434b99268848809f3453 2024-11-28T03:41:15,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1eeb194d63e3434b99268848809f3453, entries=150, sequenceid=530, filesize=12.0 K 2024-11-28T03:41:15,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/c011b4f51eeb4490a843ad2a62054414 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/c011b4f51eeb4490a843ad2a62054414 2024-11-28T03:41:15,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/c011b4f51eeb4490a843ad2a62054414, entries=150, sequenceid=530, filesize=12.0 K 2024-11-28T03:41:15,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b7a40554a92e84435d4dc19dc444ec8c in 962ms, sequenceid=530, compaction requested=true 2024-11-28T03:41:15,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:15,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:15,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:15,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:15,897 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:15,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:15,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:15,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T03:41:15,898 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:15,899 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:15,899 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:15,899 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,900 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da30e89360474b69981f2495c65341bc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/62d32d8172dc4f3a93af1665d1f2d2c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1eeb194d63e3434b99268848809f3453] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=37.2 K 2024-11-28T03:41:15,900 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:15,900 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:15,900 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:15,900 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a277f8eb0dc54508ba6f6adbaea8e93c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/88e4db5212d34b54978e3b7c7d06d881, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/e11555900c274c85b71e2333ac3b6d41] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=44.3 K 2024-11-28T03:41:15,901 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting a277f8eb0dc54508ba6f6adbaea8e93c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732765273775 2024-11-28T03:41:15,901 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting da30e89360474b69981f2495c65341bc, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732765273775 2024-11-28T03:41:15,901 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 88e4db5212d34b54978e3b7c7d06d881, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1732765274451 2024-11-28T03:41:15,902 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62d32d8172dc4f3a93af1665d1f2d2c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1732765274451 2024-11-28T03:41:15,903 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e11555900c274c85b71e2333ac3b6d41, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732765274803 2024-11-28T03:41:15,904 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eeb194d63e3434b99268848809f3453, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732765274803 2024-11-28T03:41:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T03:41:15,934 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:15,935 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/5e5e9be540c64e84a9f55406af02ae1b is 50, key is test_row_0/B:col10/1732765274803/Put/seqid=0 2024-11-28T03:41:15,960 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#112 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:15,961 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/b5745f7de92d4847a9978b3fe4a0962f is 50, key is test_row_0/A:col10/1732765274803/Put/seqid=0 2024-11-28T03:41:16,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741948_1124 (size=13561) 2024-11-28T03:41:16,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T03:41:16,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:16,029 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:41:16,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:16,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:16,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:16,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:16,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:16,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741949_1125 (size=13561) 2024-11-28T03:41:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/da294e80fe234316b1549bf3df86ca6b is 50, key is test_row_0/A:col10/1732765274973/Put/seqid=0 2024-11-28T03:41:16,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741950_1126 (size=12301) 2024-11-28T03:41:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:16,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. as already flushing 2024-11-28T03:41:16,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765336190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765336192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765336192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765336196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,249 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-28T03:41:16,249 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:16,250 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-28T03:41:16,250 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:16,254 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-28T03:41:16,254 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:16,255 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:16,255 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:16,255 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:16,255 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:16,255 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:16,256 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-28T03:41:16,256 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:16,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:16,256 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:16,256 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:16,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:16,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:16,258 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:16,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:16,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765336306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765336306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765336306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765336317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,425 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/5e5e9be540c64e84a9f55406af02ae1b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/5e5e9be540c64e84a9f55406af02ae1b 2024-11-28T03:41:16,431 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into 5e5e9be540c64e84a9f55406af02ae1b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:16,431 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:16,431 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=13, startTime=1732765275897; duration=0sec 2024-11-28T03:41:16,431 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:16,432 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:16,432 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:16,433 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:16,433 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:16,433 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:16,433 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ead034adcab49abb200e226e0f33671, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b9727f0b6f3641ebae89d1956310b343, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/c011b4f51eeb4490a843ad2a62054414] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=37.2 K 2024-11-28T03:41:16,434 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ead034adcab49abb200e226e0f33671, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732765273775 2024-11-28T03:41:16,434 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9727f0b6f3641ebae89d1956310b343, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1732765274451 2024-11-28T03:41:16,435 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c011b4f51eeb4490a843ad2a62054414, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732765274803 2024-11-28T03:41:16,442 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#114 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:16,443 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/f02a20928e994df0b38581e20b968006 is 50, key is test_row_0/C:col10/1732765274803/Put/seqid=0 2024-11-28T03:41:16,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741951_1127 (size=13561) 2024-11-28T03:41:16,452 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/b5745f7de92d4847a9978b3fe4a0962f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b5745f7de92d4847a9978b3fe4a0962f 2024-11-28T03:41:16,458 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/f02a20928e994df0b38581e20b968006 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f02a20928e994df0b38581e20b968006 2024-11-28T03:41:16,460 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into b5745f7de92d4847a9978b3fe4a0962f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:16,460 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:16,460 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=13, startTime=1732765275897; duration=0sec 2024-11-28T03:41:16,461 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:16,461 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:16,465 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into f02a20928e994df0b38581e20b968006(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:16,465 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:16,465 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=13, startTime=1732765275897; duration=0sec 2024-11-28T03:41:16,465 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:16,465 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:16,505 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/da294e80fe234316b1549bf3df86ca6b 2024-11-28T03:41:16,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/fa9804f9cd474356a43b4bc9c3b7b1b8 is 50, key is test_row_0/B:col10/1732765274973/Put/seqid=0 2024-11-28T03:41:16,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765336517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765336516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765336517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765336526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741952_1128 (size=12301) 2024-11-28T03:41:16,529 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/fa9804f9cd474356a43b4bc9c3b7b1b8 2024-11-28T03:41:16,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/f311743056ff42078d65ee6e71f950f2 is 50, key is test_row_0/C:col10/1732765274973/Put/seqid=0 2024-11-28T03:41:16,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741953_1129 (size=12301) 2024-11-28T03:41:16,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42680 deadline: 1732765336827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732765336827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42684 deadline: 1732765336827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42708 deadline: 1732765336836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42700 deadline: 1732765336836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,837 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in b7a40554a92e84435d4dc19dc444ec8c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8299 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:41:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T03:41:16,951 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/f311743056ff42078d65ee6e71f950f2 2024-11-28T03:41:16,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/da294e80fe234316b1549bf3df86ca6b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/da294e80fe234316b1549bf3df86ca6b 2024-11-28T03:41:16,961 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/da294e80fe234316b1549bf3df86ca6b, entries=150, sequenceid=549, filesize=12.0 K 2024-11-28T03:41:16,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/fa9804f9cd474356a43b4bc9c3b7b1b8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fa9804f9cd474356a43b4bc9c3b7b1b8 2024-11-28T03:41:16,966 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fa9804f9cd474356a43b4bc9c3b7b1b8, entries=150, sequenceid=549, filesize=12.0 K 2024-11-28T03:41:16,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/f311743056ff42078d65ee6e71f950f2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f311743056ff42078d65ee6e71f950f2 2024-11-28T03:41:16,971 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f311743056ff42078d65ee6e71f950f2, entries=150, sequenceid=549, filesize=12.0 K 2024-11-28T03:41:16,972 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b7a40554a92e84435d4dc19dc444ec8c in 943ms, sequenceid=549, compaction requested=false 2024-11-28T03:41:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-28T03:41:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-28T03:41:16,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-28T03:41:16,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1910 sec 2024-11-28T03:41:16,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.1980 sec 2024-11-28T03:41:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:17,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T03:41:17,339 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-28T03:41:17,339 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-28T03:41:17,339 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-28T03:41:17,339 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:17,339 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:17,339 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:17,339 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:17,339 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:17,339 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:17,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:17,339 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:17,339 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:17,339 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:17,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:17,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:17,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:17,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:17,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:17,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:17,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:17,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:17,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/23155dac182a4a12bf29e535c0344115 is 50, key is test_row_0/A:col10/1732765277336/Put/seqid=0 2024-11-28T03:41:17,348 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-28T03:41:17,348 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:17,348 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:17,348 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:17,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:17,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741954_1130 (size=12301) 2024-11-28T03:41:17,604 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T03:41:17,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/23155dac182a4a12bf29e535c0344115 2024-11-28T03:41:17,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/703c88889cd14bd4aa057bfb323ecdb2 is 50, key is test_row_0/B:col10/1732765277336/Put/seqid=0 2024-11-28T03:41:17,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741955_1131 (size=12301) 2024-11-28T03:41:18,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/703c88889cd14bd4aa057bfb323ecdb2 2024-11-28T03:41:18,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/18fd175dc19144f793df1a06fa85290d is 50, key is test_row_0/C:col10/1732765277336/Put/seqid=0 2024-11-28T03:41:18,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741956_1132 (size=12301) 2024-11-28T03:41:18,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/18fd175dc19144f793df1a06fa85290d 2024-11-28T03:41:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/23155dac182a4a12bf29e535c0344115 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/23155dac182a4a12bf29e535c0344115 2024-11-28T03:41:18,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/23155dac182a4a12bf29e535c0344115, entries=150, sequenceid=569, filesize=12.0 K 2024-11-28T03:41:18,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/703c88889cd14bd4aa057bfb323ecdb2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/703c88889cd14bd4aa057bfb323ecdb2 2024-11-28T03:41:18,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/703c88889cd14bd4aa057bfb323ecdb2, entries=150, sequenceid=569, filesize=12.0 K 2024-11-28T03:41:18,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/18fd175dc19144f793df1a06fa85290d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/18fd175dc19144f793df1a06fa85290d 2024-11-28T03:41:18,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/18fd175dc19144f793df1a06fa85290d, entries=150, sequenceid=569, filesize=12.0 K 2024-11-28T03:41:18,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for b7a40554a92e84435d4dc19dc444ec8c in 858ms, sequenceid=569, compaction requested=true 2024-11-28T03:41:18,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:18,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:18,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:18,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:18,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:18,197 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:18,197 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:18,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a40554a92e84435d4dc19dc444ec8c:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:18,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:18,198 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:18,198 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/B is initiating minor compaction (all files) 2024-11-28T03:41:18,198 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:18,198 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/B in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:18,198 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/A is initiating minor compaction (all files) 2024-11-28T03:41:18,198 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/5e5e9be540c64e84a9f55406af02ae1b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fa9804f9cd474356a43b4bc9c3b7b1b8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/703c88889cd14bd4aa057bfb323ecdb2] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=37.3 K 2024-11-28T03:41:18,198 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/A in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:18,198 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b5745f7de92d4847a9978b3fe4a0962f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/da294e80fe234316b1549bf3df86ca6b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/23155dac182a4a12bf29e535c0344115] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=37.3 K 2024-11-28T03:41:18,198 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e5e9be540c64e84a9f55406af02ae1b, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732765274803 2024-11-28T03:41:18,199 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5745f7de92d4847a9978b3fe4a0962f, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732765274803 2024-11-28T03:41:18,199 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9804f9cd474356a43b4bc9c3b7b1b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1732765274973 2024-11-28T03:41:18,199 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting da294e80fe234316b1549bf3df86ca6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1732765274973 2024-11-28T03:41:18,199 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 703c88889cd14bd4aa057bfb323ecdb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1732765276167 2024-11-28T03:41:18,199 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23155dac182a4a12bf29e535c0344115, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1732765276167 2024-11-28T03:41:18,214 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#A#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:18,215 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/eba2d80eecd9436ca90d1273474e8460 is 50, key is test_row_0/A:col10/1732765277336/Put/seqid=0 2024-11-28T03:41:18,222 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#B#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:18,223 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/47029c7686c041dbb0c882a760444218 is 50, key is test_row_0/B:col10/1732765277336/Put/seqid=0 2024-11-28T03:41:18,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741957_1133 (size=13663) 2024-11-28T03:41:18,239 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/eba2d80eecd9436ca90d1273474e8460 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/eba2d80eecd9436ca90d1273474e8460 2024-11-28T03:41:18,246 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/A of b7a40554a92e84435d4dc19dc444ec8c into eba2d80eecd9436ca90d1273474e8460(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:18,246 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:18,246 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/A, priority=13, startTime=1732765278197; duration=0sec 2024-11-28T03:41:18,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741958_1134 (size=13663) 2024-11-28T03:41:18,246 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:18,246 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:A 2024-11-28T03:41:18,246 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:18,248 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:18,248 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): b7a40554a92e84435d4dc19dc444ec8c/C is initiating minor compaction (all files) 2024-11-28T03:41:18,248 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a40554a92e84435d4dc19dc444ec8c/C in TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:18,248 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f02a20928e994df0b38581e20b968006, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f311743056ff42078d65ee6e71f950f2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/18fd175dc19144f793df1a06fa85290d] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp, totalSize=37.3 K 2024-11-28T03:41:18,248 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f02a20928e994df0b38581e20b968006, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732765274803 2024-11-28T03:41:18,249 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f311743056ff42078d65ee6e71f950f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1732765274973 2024-11-28T03:41:18,249 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18fd175dc19144f793df1a06fa85290d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1732765276167 2024-11-28T03:41:18,252 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/47029c7686c041dbb0c882a760444218 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/47029c7686c041dbb0c882a760444218 2024-11-28T03:41:18,259 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/B of b7a40554a92e84435d4dc19dc444ec8c into 47029c7686c041dbb0c882a760444218(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:18,259 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:18,259 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/B, priority=13, startTime=1732765278197; duration=0sec 2024-11-28T03:41:18,259 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:18,259 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:B 2024-11-28T03:41:18,261 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a40554a92e84435d4dc19dc444ec8c#C#compaction#122 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:18,262 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/a3ec397e0f3c4d4fafdb13ed10ac6d6e is 50, key is test_row_0/C:col10/1732765277336/Put/seqid=0 2024-11-28T03:41:18,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741959_1135 (size=13663) 2024-11-28T03:41:18,680 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/a3ec397e0f3c4d4fafdb13ed10ac6d6e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/a3ec397e0f3c4d4fafdb13ed10ac6d6e 2024-11-28T03:41:18,689 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a40554a92e84435d4dc19dc444ec8c/C of b7a40554a92e84435d4dc19dc444ec8c into a3ec397e0f3c4d4fafdb13ed10ac6d6e(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:18,689 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a40554a92e84435d4dc19dc444ec8c: 2024-11-28T03:41:18,689 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., storeName=b7a40554a92e84435d4dc19dc444ec8c/C, priority=13, startTime=1732765278197; duration=0sec 2024-11-28T03:41:18,690 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:18,690 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a40554a92e84435d4dc19dc444ec8c:C 2024-11-28T03:41:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T03:41:18,947 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:26,909 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-28T03:41:26,909 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:26,909 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:26,910 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:26,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:26,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T03:41:26,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-28T03:41:26,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-28T03:41:26,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-28T03:41:26,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 128 2024-11-28T03:41:26,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-28T03:41:26,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T03:41:26,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2197 2024-11-28T03:41:26,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2232 2024-11-28T03:41:26,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T03:41:26,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1050 2024-11-28T03:41:26,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3150 rows 2024-11-28T03:41:26,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1051 2024-11-28T03:41:26,911 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3152 rows 2024-11-28T03:41:26,911 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:41:26,911 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:41:26,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:26,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:26,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:26,916 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T03:41:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T03:41:26,925 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765286924"}]},"ts":"1732765286924"} 2024-11-28T03:41:26,927 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T03:41:26,927 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T03:41:26,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T03:41:26,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a40554a92e84435d4dc19dc444ec8c, UNASSIGN}] 2024-11-28T03:41:26,936 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a40554a92e84435d4dc19dc444ec8c, UNASSIGN 2024-11-28T03:41:26,937 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b7a40554a92e84435d4dc19dc444ec8c, regionState=CLOSING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:41:26,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a40554a92e84435d4dc19dc444ec8c, UNASSIGN because future has completed 2024-11-28T03:41:26,941 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T03:41:26,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:41:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T03:41:27,099 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:27,099 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T03:41:27,100 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing b7a40554a92e84435d4dc19dc444ec8c, disabling compactions & flushes 2024-11-28T03:41:27,100 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:27,100 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:27,100 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. after waiting 0 ms 2024-11-28T03:41:27,100 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:27,100 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing b7a40554a92e84435d4dc19dc444ec8c 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-28T03:41:27,100 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=A 2024-11-28T03:41:27,100 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:27,100 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=B 2024-11-28T03:41:27,100 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:27,100 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a40554a92e84435d4dc19dc444ec8c, store=C 2024-11-28T03:41:27,100 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:27,105 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/1a1faae4babd43acbabb9914b3945d9b is 50, key is test_row_1/A:col10/1732765286907/Put/seqid=0 2024-11-28T03:41:27,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741960_1136 (size=7415) 2024-11-28T03:41:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T03:41:27,510 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/1a1faae4babd43acbabb9914b3945d9b 2024-11-28T03:41:27,518 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/2c6ba15cc4cd4b759c5eba3c82f5ceb0 is 50, key is test_row_1/B:col10/1732765286907/Put/seqid=0 2024-11-28T03:41:27,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741961_1137 (size=7415) 2024-11-28T03:41:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T03:41:27,923 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/2c6ba15cc4cd4b759c5eba3c82f5ceb0 2024-11-28T03:41:27,932 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/367e129a9e984483a9ba6a945a61ddf2 is 50, key is test_row_1/C:col10/1732765286907/Put/seqid=0 2024-11-28T03:41:27,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741962_1138 (size=7415) 2024-11-28T03:41:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T03:41:28,345 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/367e129a9e984483a9ba6a945a61ddf2 2024-11-28T03:41:28,357 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/A/1a1faae4babd43acbabb9914b3945d9b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/1a1faae4babd43acbabb9914b3945d9b 2024-11-28T03:41:28,362 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/1a1faae4babd43acbabb9914b3945d9b, entries=50, sequenceid=577, filesize=7.2 K 2024-11-28T03:41:28,363 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/B/2c6ba15cc4cd4b759c5eba3c82f5ceb0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/2c6ba15cc4cd4b759c5eba3c82f5ceb0 2024-11-28T03:41:28,371 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/2c6ba15cc4cd4b759c5eba3c82f5ceb0, entries=50, sequenceid=577, filesize=7.2 K 2024-11-28T03:41:28,372 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/.tmp/C/367e129a9e984483a9ba6a945a61ddf2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/367e129a9e984483a9ba6a945a61ddf2 2024-11-28T03:41:28,382 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/367e129a9e984483a9ba6a945a61ddf2, entries=50, sequenceid=577, filesize=7.2 K 2024-11-28T03:41:28,383 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for b7a40554a92e84435d4dc19dc444ec8c in 1283ms, sequenceid=577, compaction requested=false 2024-11-28T03:41:28,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/3270cab6ae3049bb9649e40701cf9a93, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc34effed35d4b8091f5d3e7c31168ea, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/73fa8883946f4fa8a45bac0ad92710fc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7b0d99fb7fec4b2daa4891d6834bae91, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8f54f6801f7f4206852880067ea1472f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a24e455d30f1457c89bc69169dbf43c2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8a628bbeef5f48a58b7840e61fd6c819, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4e2d02c73c9e410dbadf6552152fb5a1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/521724a408494ea08d19411bdedbc58e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/ee2ca06289e74abf86fbabf466ee9a3e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/66c8cfaeffb44ee2827e714d90131ad1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/43bf11a9ddee4e50b6cbc3fdf93e1bf3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7ed4298656364e41841f646d8c4eeb73, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/2d8f849e807241c3b37b51ec6710ea87, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/5d1f4fcc40c5457cb14f0e496d4282aa, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b4a7cb714fd642e6828832808e015a18, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7dc3fc4cc6574a2abdcf61eb911e9e00, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8c1cce45660840b499495bb1e1de7f7e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b6744f3a18e84a9b94a82b013f5cfc5c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/c4f9bebb2302433ebf4ac02d73288287, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7853925f2ec64e169729c439cc50750d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/d063dfdf45df489090902e5e0ad4a920, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4abab88d47fa4ff2ad57edc2e8b6ba6b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/61cf5d93b4834d0499aa5b68e9ee85d3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/21e330783b724a5a83b36a9275fe31bf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4282d0aad6dc4c9c9c2f3e6765eeee4a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/626c1653490b464cb2c322c2044b5563, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/300f97c3fc014ced88de62e1087f4934, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cb976ccb71cb4dbbbabed04cbfadbe3d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/6032b30a4f004df0905d8585f7bcf64d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/422aae67049d484db3a62b68a599169f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/82f61dc5fef44668903e4f5a3325c50a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/f2f22210de7d45338779a394fbf32c1e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc9566afe4144b7c861f91a057ecf56a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a277f8eb0dc54508ba6f6adbaea8e93c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/88e4db5212d34b54978e3b7c7d06d881, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/e11555900c274c85b71e2333ac3b6d41, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b5745f7de92d4847a9978b3fe4a0962f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/da294e80fe234316b1549bf3df86ca6b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/23155dac182a4a12bf29e535c0344115] to archive 2024-11-28T03:41:28,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:41:28,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/3270cab6ae3049bb9649e40701cf9a93 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/3270cab6ae3049bb9649e40701cf9a93 2024-11-28T03:41:28,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc34effed35d4b8091f5d3e7c31168ea to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc34effed35d4b8091f5d3e7c31168ea 2024-11-28T03:41:28,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/73fa8883946f4fa8a45bac0ad92710fc to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/73fa8883946f4fa8a45bac0ad92710fc 2024-11-28T03:41:28,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7b0d99fb7fec4b2daa4891d6834bae91 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7b0d99fb7fec4b2daa4891d6834bae91 2024-11-28T03:41:28,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8f54f6801f7f4206852880067ea1472f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8f54f6801f7f4206852880067ea1472f 2024-11-28T03:41:28,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a24e455d30f1457c89bc69169dbf43c2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a24e455d30f1457c89bc69169dbf43c2 2024-11-28T03:41:28,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8a628bbeef5f48a58b7840e61fd6c819 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8a628bbeef5f48a58b7840e61fd6c819 2024-11-28T03:41:28,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4e2d02c73c9e410dbadf6552152fb5a1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4e2d02c73c9e410dbadf6552152fb5a1 2024-11-28T03:41:28,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/521724a408494ea08d19411bdedbc58e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/521724a408494ea08d19411bdedbc58e 2024-11-28T03:41:28,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/ee2ca06289e74abf86fbabf466ee9a3e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/ee2ca06289e74abf86fbabf466ee9a3e 2024-11-28T03:41:28,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/66c8cfaeffb44ee2827e714d90131ad1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/66c8cfaeffb44ee2827e714d90131ad1 2024-11-28T03:41:28,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/43bf11a9ddee4e50b6cbc3fdf93e1bf3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/43bf11a9ddee4e50b6cbc3fdf93e1bf3 2024-11-28T03:41:28,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7ed4298656364e41841f646d8c4eeb73 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7ed4298656364e41841f646d8c4eeb73 2024-11-28T03:41:28,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/2d8f849e807241c3b37b51ec6710ea87 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/2d8f849e807241c3b37b51ec6710ea87 2024-11-28T03:41:28,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/5d1f4fcc40c5457cb14f0e496d4282aa to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/5d1f4fcc40c5457cb14f0e496d4282aa 2024-11-28T03:41:28,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b4a7cb714fd642e6828832808e015a18 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b4a7cb714fd642e6828832808e015a18 2024-11-28T03:41:28,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7dc3fc4cc6574a2abdcf61eb911e9e00 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7dc3fc4cc6574a2abdcf61eb911e9e00 2024-11-28T03:41:28,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8c1cce45660840b499495bb1e1de7f7e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/8c1cce45660840b499495bb1e1de7f7e 2024-11-28T03:41:28,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b6744f3a18e84a9b94a82b013f5cfc5c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b6744f3a18e84a9b94a82b013f5cfc5c 2024-11-28T03:41:28,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/c4f9bebb2302433ebf4ac02d73288287 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/c4f9bebb2302433ebf4ac02d73288287 2024-11-28T03:41:28,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7853925f2ec64e169729c439cc50750d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/7853925f2ec64e169729c439cc50750d 2024-11-28T03:41:28,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/d063dfdf45df489090902e5e0ad4a920 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/d063dfdf45df489090902e5e0ad4a920 2024-11-28T03:41:28,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4abab88d47fa4ff2ad57edc2e8b6ba6b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4abab88d47fa4ff2ad57edc2e8b6ba6b 2024-11-28T03:41:28,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/61cf5d93b4834d0499aa5b68e9ee85d3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/61cf5d93b4834d0499aa5b68e9ee85d3 2024-11-28T03:41:28,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/21e330783b724a5a83b36a9275fe31bf to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/21e330783b724a5a83b36a9275fe31bf 2024-11-28T03:41:28,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4282d0aad6dc4c9c9c2f3e6765eeee4a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/4282d0aad6dc4c9c9c2f3e6765eeee4a 2024-11-28T03:41:28,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/626c1653490b464cb2c322c2044b5563 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/626c1653490b464cb2c322c2044b5563 2024-11-28T03:41:28,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/300f97c3fc014ced88de62e1087f4934 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/300f97c3fc014ced88de62e1087f4934 2024-11-28T03:41:28,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cb976ccb71cb4dbbbabed04cbfadbe3d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cb976ccb71cb4dbbbabed04cbfadbe3d 2024-11-28T03:41:28,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/6032b30a4f004df0905d8585f7bcf64d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/6032b30a4f004df0905d8585f7bcf64d 2024-11-28T03:41:28,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/422aae67049d484db3a62b68a599169f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/422aae67049d484db3a62b68a599169f 2024-11-28T03:41:28,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/82f61dc5fef44668903e4f5a3325c50a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/82f61dc5fef44668903e4f5a3325c50a 2024-11-28T03:41:28,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/f2f22210de7d45338779a394fbf32c1e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/f2f22210de7d45338779a394fbf32c1e 2024-11-28T03:41:28,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc9566afe4144b7c861f91a057ecf56a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/cc9566afe4144b7c861f91a057ecf56a 2024-11-28T03:41:28,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a277f8eb0dc54508ba6f6adbaea8e93c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/a277f8eb0dc54508ba6f6adbaea8e93c 2024-11-28T03:41:28,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/88e4db5212d34b54978e3b7c7d06d881 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/88e4db5212d34b54978e3b7c7d06d881 2024-11-28T03:41:28,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/e11555900c274c85b71e2333ac3b6d41 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/e11555900c274c85b71e2333ac3b6d41 2024-11-28T03:41:28,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b5745f7de92d4847a9978b3fe4a0962f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/b5745f7de92d4847a9978b3fe4a0962f 2024-11-28T03:41:28,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/da294e80fe234316b1549bf3df86ca6b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/da294e80fe234316b1549bf3df86ca6b 2024-11-28T03:41:28,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/23155dac182a4a12bf29e535c0344115 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/23155dac182a4a12bf29e535c0344115 2024-11-28T03:41:28,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1288ffc549cb401e923165948ad5239e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1156b9c960c048b29230f4b0fc8780e5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/0ff513ecb820405e8fb69c87ba875b9e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ff7d4707b1264258b604906a03126654, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da9f5cdec8a84b0eb12bd4c136d56a07, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a217043499114ab6a9fe41c955f52fb1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/03b7843892eb488bb8c7afa3be62cca3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/c746e3209d324c1881b221b3a0b21e3c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/71e624d104b341fda559171e38572fb1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6925bca5adaa44c88975da79da454cbe, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7399a14b984248e187b2aac60bf6631a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/279c6526c1fa4360994c4f12f93881a0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7552475f542a4a1c95312412280aaba7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/00d75a73f08f4ef1ac30f64aab9f159e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ce46a0236f71424dbf6242d2831b743a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/11e1b1fb1cce4272a9eef321193560b9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3754df1b358d41b49258482662f600c7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fe7cd041ab4e4616a466d71d06a6d537, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/f2ec049816a245bd8e2953169222c0b3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6916a5c6863946acab4f19abdf19f972, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/9bd6b939c7994b23be6fc6fb0957a623, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/809abbc8df784c3db58b109293b5df93, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ecbd7f23ede64ab2aa387fa4c832c4a1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/2f4e5954744e41e6a23666faf8565488, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/d7b053c597454a0f8634f103b4b16169, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/bec7a1e131e74f6a8f94187c02ec5518, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/48830e660a2f4095a77a69ab05193e55, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cb103e0b3fa4255873cda52361332f3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/071f3cd480fb4f4baff8bb2d8f034cac, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a54f4cdbd99e4792837f5b461deb1452, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cd3f15527a1413e85a176fcdcb3968c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dd8c05fafe3b4754aea5d19038b024f9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/292ca14fc59f4ee38e010766d6a3f3ab, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da30e89360474b69981f2495c65341bc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dbb5c56c3426471d8f6022d0e48fee9b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/62d32d8172dc4f3a93af1665d1f2d2c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/5e5e9be540c64e84a9f55406af02ae1b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1eeb194d63e3434b99268848809f3453, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fa9804f9cd474356a43b4bc9c3b7b1b8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/703c88889cd14bd4aa057bfb323ecdb2] to archive 2024-11-28T03:41:28,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:41:28,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1288ffc549cb401e923165948ad5239e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1288ffc549cb401e923165948ad5239e 2024-11-28T03:41:28,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1156b9c960c048b29230f4b0fc8780e5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1156b9c960c048b29230f4b0fc8780e5 2024-11-28T03:41:28,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/0ff513ecb820405e8fb69c87ba875b9e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/0ff513ecb820405e8fb69c87ba875b9e 2024-11-28T03:41:28,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ff7d4707b1264258b604906a03126654 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ff7d4707b1264258b604906a03126654 2024-11-28T03:41:28,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da9f5cdec8a84b0eb12bd4c136d56a07 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da9f5cdec8a84b0eb12bd4c136d56a07 2024-11-28T03:41:28,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a217043499114ab6a9fe41c955f52fb1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a217043499114ab6a9fe41c955f52fb1 2024-11-28T03:41:28,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/03b7843892eb488bb8c7afa3be62cca3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/03b7843892eb488bb8c7afa3be62cca3 2024-11-28T03:41:28,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/c746e3209d324c1881b221b3a0b21e3c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/c746e3209d324c1881b221b3a0b21e3c 2024-11-28T03:41:28,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/71e624d104b341fda559171e38572fb1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/71e624d104b341fda559171e38572fb1 2024-11-28T03:41:28,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6925bca5adaa44c88975da79da454cbe to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6925bca5adaa44c88975da79da454cbe 2024-11-28T03:41:28,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7399a14b984248e187b2aac60bf6631a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7399a14b984248e187b2aac60bf6631a 2024-11-28T03:41:28,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/279c6526c1fa4360994c4f12f93881a0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/279c6526c1fa4360994c4f12f93881a0 2024-11-28T03:41:28,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7552475f542a4a1c95312412280aaba7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/7552475f542a4a1c95312412280aaba7 2024-11-28T03:41:28,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/00d75a73f08f4ef1ac30f64aab9f159e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/00d75a73f08f4ef1ac30f64aab9f159e 2024-11-28T03:41:28,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ce46a0236f71424dbf6242d2831b743a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ce46a0236f71424dbf6242d2831b743a 2024-11-28T03:41:28,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/11e1b1fb1cce4272a9eef321193560b9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/11e1b1fb1cce4272a9eef321193560b9 2024-11-28T03:41:28,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3754df1b358d41b49258482662f600c7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3754df1b358d41b49258482662f600c7 2024-11-28T03:41:28,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fe7cd041ab4e4616a466d71d06a6d537 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fe7cd041ab4e4616a466d71d06a6d537 2024-11-28T03:41:28,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/f2ec049816a245bd8e2953169222c0b3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/f2ec049816a245bd8e2953169222c0b3 2024-11-28T03:41:28,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6916a5c6863946acab4f19abdf19f972 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/6916a5c6863946acab4f19abdf19f972 2024-11-28T03:41:28,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/9bd6b939c7994b23be6fc6fb0957a623 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/9bd6b939c7994b23be6fc6fb0957a623 2024-11-28T03:41:28,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/809abbc8df784c3db58b109293b5df93 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/809abbc8df784c3db58b109293b5df93 2024-11-28T03:41:28,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ecbd7f23ede64ab2aa387fa4c832c4a1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/ecbd7f23ede64ab2aa387fa4c832c4a1 2024-11-28T03:41:28,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/2f4e5954744e41e6a23666faf8565488 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/2f4e5954744e41e6a23666faf8565488 2024-11-28T03:41:28,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/d7b053c597454a0f8634f103b4b16169 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/d7b053c597454a0f8634f103b4b16169 2024-11-28T03:41:28,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/bec7a1e131e74f6a8f94187c02ec5518 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/bec7a1e131e74f6a8f94187c02ec5518 2024-11-28T03:41:28,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/48830e660a2f4095a77a69ab05193e55 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/48830e660a2f4095a77a69ab05193e55 2024-11-28T03:41:28,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cb103e0b3fa4255873cda52361332f3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cb103e0b3fa4255873cda52361332f3 2024-11-28T03:41:28,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/071f3cd480fb4f4baff8bb2d8f034cac to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/071f3cd480fb4f4baff8bb2d8f034cac 2024-11-28T03:41:28,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a54f4cdbd99e4792837f5b461deb1452 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/a54f4cdbd99e4792837f5b461deb1452 2024-11-28T03:41:28,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cd3f15527a1413e85a176fcdcb3968c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/3cd3f15527a1413e85a176fcdcb3968c 2024-11-28T03:41:28,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dd8c05fafe3b4754aea5d19038b024f9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dd8c05fafe3b4754aea5d19038b024f9 2024-11-28T03:41:28,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/292ca14fc59f4ee38e010766d6a3f3ab to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/292ca14fc59f4ee38e010766d6a3f3ab 2024-11-28T03:41:28,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da30e89360474b69981f2495c65341bc to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/da30e89360474b69981f2495c65341bc 2024-11-28T03:41:28,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dbb5c56c3426471d8f6022d0e48fee9b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/dbb5c56c3426471d8f6022d0e48fee9b 2024-11-28T03:41:28,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/62d32d8172dc4f3a93af1665d1f2d2c4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/62d32d8172dc4f3a93af1665d1f2d2c4 2024-11-28T03:41:28,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/5e5e9be540c64e84a9f55406af02ae1b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/5e5e9be540c64e84a9f55406af02ae1b 2024-11-28T03:41:28,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1eeb194d63e3434b99268848809f3453 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/1eeb194d63e3434b99268848809f3453 2024-11-28T03:41:28,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fa9804f9cd474356a43b4bc9c3b7b1b8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/fa9804f9cd474356a43b4bc9c3b7b1b8 2024-11-28T03:41:28,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/703c88889cd14bd4aa057bfb323ecdb2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/703c88889cd14bd4aa057bfb323ecdb2 2024-11-28T03:41:28,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/04b9f19a1bd24a9a9b3bd6cde1a6a281, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/17d753e28429406d84ba56d23ad02995, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/54205189e8354dd1a21eaea032689b7a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0ecd996e7775440f9d13b20e0d56a7b2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/69282aee0fc3434d99cf91af777f88c3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/32b053977ad144918d448a757af85bf6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5c6621a6d87f4346becee382e2d20620, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e29bf4f302374362af566a3be94624fa, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/28358edcdcbf41b6b6f4b8b96b9d68bc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/12d3d4b79113485f8511ce944f04725f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5510e3577fe44e3fb1d595dccb680b7e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0bea626a584148899c94534065519f76, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e62619ff9ec34cee9b5e7d199203ac28, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d4012def3064408fadb9656f7c120639, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/98c09ba65c4349e18d84260f8fb27364, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/cc02c0c1390d400498a13cc493154cf5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3b347fe6dd5a4a12a9e3d20b58bb6cd0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7bb4cea39b37444f979b62fe0f3c9251, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/670dd8e8012b43bcac20d1b666077be8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/82ffe6d05521429aa7997893bee483a7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d2cc516f8f5d4da99c6f51d65eae3dba, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/9099f69f5e214e0ca5c07af7465de646, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e66090b5f6224fe3ba9e96e7ec5d33d1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3d9fb110393d4cf59684d5fb94bfcee1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/541b6bd1740249ec835d6f0eca828dd1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e186ecd23d034843a3bbae371fa46d92, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/faf50454bd154698ac727993fc3622b7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/45c4520471324514a7ffd7768d23258b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/15e54d33621b47c18e4c2adf732b5232, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/277ae4b675674bce9776e3a080fdfe77, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/fffc7fc6dc054bea80008d3483d35366, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/74166a4c95134f7eb68e312a4b301b74, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ef58fa3b970439a974a7f6c9fdb11c0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ead034adcab49abb200e226e0f33671, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b059235207044c1999b61ae0acc474f4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b9727f0b6f3641ebae89d1956310b343, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f02a20928e994df0b38581e20b968006, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/c011b4f51eeb4490a843ad2a62054414, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f311743056ff42078d65ee6e71f950f2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/18fd175dc19144f793df1a06fa85290d] to archive 2024-11-28T03:41:28,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:41:28,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/04b9f19a1bd24a9a9b3bd6cde1a6a281 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/04b9f19a1bd24a9a9b3bd6cde1a6a281 2024-11-28T03:41:28,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/17d753e28429406d84ba56d23ad02995 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/17d753e28429406d84ba56d23ad02995 2024-11-28T03:41:28,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/54205189e8354dd1a21eaea032689b7a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/54205189e8354dd1a21eaea032689b7a 2024-11-28T03:41:28,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0ecd996e7775440f9d13b20e0d56a7b2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0ecd996e7775440f9d13b20e0d56a7b2 2024-11-28T03:41:28,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/69282aee0fc3434d99cf91af777f88c3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/69282aee0fc3434d99cf91af777f88c3 2024-11-28T03:41:28,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/32b053977ad144918d448a757af85bf6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/32b053977ad144918d448a757af85bf6 2024-11-28T03:41:28,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5c6621a6d87f4346becee382e2d20620 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5c6621a6d87f4346becee382e2d20620 2024-11-28T03:41:28,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e29bf4f302374362af566a3be94624fa to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e29bf4f302374362af566a3be94624fa 2024-11-28T03:41:28,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/28358edcdcbf41b6b6f4b8b96b9d68bc to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/28358edcdcbf41b6b6f4b8b96b9d68bc 2024-11-28T03:41:28,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/12d3d4b79113485f8511ce944f04725f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/12d3d4b79113485f8511ce944f04725f 2024-11-28T03:41:28,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5510e3577fe44e3fb1d595dccb680b7e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/5510e3577fe44e3fb1d595dccb680b7e 2024-11-28T03:41:28,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0bea626a584148899c94534065519f76 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/0bea626a584148899c94534065519f76 2024-11-28T03:41:28,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e62619ff9ec34cee9b5e7d199203ac28 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e62619ff9ec34cee9b5e7d199203ac28 2024-11-28T03:41:28,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d4012def3064408fadb9656f7c120639 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d4012def3064408fadb9656f7c120639 2024-11-28T03:41:28,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/98c09ba65c4349e18d84260f8fb27364 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/98c09ba65c4349e18d84260f8fb27364 2024-11-28T03:41:28,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/cc02c0c1390d400498a13cc493154cf5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/cc02c0c1390d400498a13cc493154cf5 2024-11-28T03:41:28,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3b347fe6dd5a4a12a9e3d20b58bb6cd0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3b347fe6dd5a4a12a9e3d20b58bb6cd0 2024-11-28T03:41:28,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7bb4cea39b37444f979b62fe0f3c9251 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7bb4cea39b37444f979b62fe0f3c9251 2024-11-28T03:41:28,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/670dd8e8012b43bcac20d1b666077be8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/670dd8e8012b43bcac20d1b666077be8 2024-11-28T03:41:28,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/82ffe6d05521429aa7997893bee483a7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/82ffe6d05521429aa7997893bee483a7 2024-11-28T03:41:28,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d2cc516f8f5d4da99c6f51d65eae3dba to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/d2cc516f8f5d4da99c6f51d65eae3dba 2024-11-28T03:41:28,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/9099f69f5e214e0ca5c07af7465de646 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/9099f69f5e214e0ca5c07af7465de646 2024-11-28T03:41:28,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e66090b5f6224fe3ba9e96e7ec5d33d1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e66090b5f6224fe3ba9e96e7ec5d33d1 2024-11-28T03:41:28,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3d9fb110393d4cf59684d5fb94bfcee1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/3d9fb110393d4cf59684d5fb94bfcee1 2024-11-28T03:41:28,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/541b6bd1740249ec835d6f0eca828dd1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/541b6bd1740249ec835d6f0eca828dd1 2024-11-28T03:41:28,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e186ecd23d034843a3bbae371fa46d92 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/e186ecd23d034843a3bbae371fa46d92 2024-11-28T03:41:28,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/faf50454bd154698ac727993fc3622b7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/faf50454bd154698ac727993fc3622b7 2024-11-28T03:41:28,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/45c4520471324514a7ffd7768d23258b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/45c4520471324514a7ffd7768d23258b 2024-11-28T03:41:28,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/15e54d33621b47c18e4c2adf732b5232 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/15e54d33621b47c18e4c2adf732b5232 2024-11-28T03:41:28,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/277ae4b675674bce9776e3a080fdfe77 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/277ae4b675674bce9776e3a080fdfe77 2024-11-28T03:41:28,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/fffc7fc6dc054bea80008d3483d35366 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/fffc7fc6dc054bea80008d3483d35366 2024-11-28T03:41:28,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/74166a4c95134f7eb68e312a4b301b74 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/74166a4c95134f7eb68e312a4b301b74 2024-11-28T03:41:28,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ef58fa3b970439a974a7f6c9fdb11c0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ef58fa3b970439a974a7f6c9fdb11c0 2024-11-28T03:41:28,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ead034adcab49abb200e226e0f33671 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/7ead034adcab49abb200e226e0f33671 2024-11-28T03:41:28,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b059235207044c1999b61ae0acc474f4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b059235207044c1999b61ae0acc474f4 2024-11-28T03:41:28,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b9727f0b6f3641ebae89d1956310b343 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/b9727f0b6f3641ebae89d1956310b343 2024-11-28T03:41:28,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f02a20928e994df0b38581e20b968006 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f02a20928e994df0b38581e20b968006 2024-11-28T03:41:28,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/c011b4f51eeb4490a843ad2a62054414 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/c011b4f51eeb4490a843ad2a62054414 2024-11-28T03:41:28,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f311743056ff42078d65ee6e71f950f2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/f311743056ff42078d65ee6e71f950f2 2024-11-28T03:41:28,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/18fd175dc19144f793df1a06fa85290d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/18fd175dc19144f793df1a06fa85290d 2024-11-28T03:41:28,691 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/recovered.edits/580.seqid, newMaxSeqId=580, maxSeqId=1 2024-11-28T03:41:28,692 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. 2024-11-28T03:41:28,693 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for b7a40554a92e84435d4dc19dc444ec8c: Waiting for close lock at 1732765287099Running coprocessor pre-close hooks at 1732765287100 (+1 ms)Disabling compacts and flushes for region at 1732765287100Disabling writes for close at 1732765287100Obtaining lock to block concurrent updates at 1732765287100Preparing flush snapshotting stores in b7a40554a92e84435d4dc19dc444ec8c at 1732765287100Finished memstore snapshotting TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732765287100Flushing stores of TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c. at 1732765287101 (+1 ms)Flushing b7a40554a92e84435d4dc19dc444ec8c/A: creating writer at 1732765287101Flushing b7a40554a92e84435d4dc19dc444ec8c/A: appending metadata at 1732765287104 (+3 ms)Flushing b7a40554a92e84435d4dc19dc444ec8c/A: closing flushed file at 1732765287104Flushing b7a40554a92e84435d4dc19dc444ec8c/B: creating writer at 1732765287515 (+411 ms)Flushing b7a40554a92e84435d4dc19dc444ec8c/B: appending metadata at 1732765287518 (+3 ms)Flushing b7a40554a92e84435d4dc19dc444ec8c/B: closing flushed file at 1732765287518Flushing b7a40554a92e84435d4dc19dc444ec8c/C: creating writer at 1732765287928 (+410 ms)Flushing b7a40554a92e84435d4dc19dc444ec8c/C: appending metadata at 1732765287932 (+4 ms)Flushing b7a40554a92e84435d4dc19dc444ec8c/C: closing flushed file at 1732765287932Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@343e68ba: reopening flushed file at 1732765288356 (+424 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b99e97f: reopening flushed file at 1732765288363 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ee574b: reopening flushed file at 1732765288371 (+8 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for b7a40554a92e84435d4dc19dc444ec8c in 1283ms, sequenceid=577, compaction requested=false at 1732765288383 (+12 ms)Writing region close event to WAL at 1732765288687 (+304 ms)Running coprocessor post-close hooks at 1732765288692 (+5 ms)Closed at 1732765288692 2024-11-28T03:41:28,696 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:28,696 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b7a40554a92e84435d4dc19dc444ec8c, regionState=CLOSED 2024-11-28T03:41:28,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:41:28,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-28T03:41:28,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure b7a40554a92e84435d4dc19dc444ec8c, server=3a546af21488,34671,1732765249447 in 1.7580 sec 2024-11-28T03:41:28,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-28T03:41:28,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a40554a92e84435d4dc19dc444ec8c, UNASSIGN in 1.7660 sec 2024-11-28T03:41:28,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-28T03:41:28,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7760 sec 2024-11-28T03:41:28,709 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765288709"}]},"ts":"1732765288709"} 2024-11-28T03:41:28,711 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T03:41:28,711 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T03:41:28,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7940 sec 2024-11-28T03:41:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T03:41:29,067 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:29,071 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T03:41:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:29,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T03:41:29,080 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:29,082 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T03:41:29,096 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:29,101 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/recovered.edits] 2024-11-28T03:41:29,106 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/1a1faae4babd43acbabb9914b3945d9b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/1a1faae4babd43acbabb9914b3945d9b 2024-11-28T03:41:29,108 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/eba2d80eecd9436ca90d1273474e8460 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/A/eba2d80eecd9436ca90d1273474e8460 2024-11-28T03:41:29,113 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/2c6ba15cc4cd4b759c5eba3c82f5ceb0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/2c6ba15cc4cd4b759c5eba3c82f5ceb0 2024-11-28T03:41:29,115 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/47029c7686c041dbb0c882a760444218 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/B/47029c7686c041dbb0c882a760444218 2024-11-28T03:41:29,119 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/367e129a9e984483a9ba6a945a61ddf2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/367e129a9e984483a9ba6a945a61ddf2 2024-11-28T03:41:29,123 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/a3ec397e0f3c4d4fafdb13ed10ac6d6e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/C/a3ec397e0f3c4d4fafdb13ed10ac6d6e 2024-11-28T03:41:29,127 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/recovered.edits/580.seqid to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c/recovered.edits/580.seqid 2024-11-28T03:41:29,128 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/b7a40554a92e84435d4dc19dc444ec8c 2024-11-28T03:41:29,128 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T03:41:29,135 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:29,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-28T03:41:29,148 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T03:41:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T03:41:29,198 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T03:41:29,199 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:29,200 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T03:41:29,200 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732765289200"}]},"ts":"9223372036854775807"} 2024-11-28T03:41:29,206 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T03:41:29,206 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b7a40554a92e84435d4dc19dc444ec8c, NAME => 'TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T03:41:29,206 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T03:41:29,206 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732765289206"}]},"ts":"9223372036854775807"} 2024-11-28T03:41:29,209 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T03:41:29,211 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:29,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 139 msec 2024-11-28T03:41:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T03:41:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T03:41:29,408 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:29,425 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=246 (was 210) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_REGION-regionserver/3a546af21488:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;3a546af21488:34671-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/3a546af21488:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_764232134_22 at /127.0.0.1:39630 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=528 (was 425) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7508 (was 7469) - AvailableMemoryMB LEAK? - 2024-11-28T03:41:29,436 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=246, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=528, ProcessCount=11, AvailableMemoryMB=7507 2024-11-28T03:41:29,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T03:41:29,438 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:41:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:29,441 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:41:29,441 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:29,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-28T03:41:29,442 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:41:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T03:41:29,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741963_1139 (size=1061) 2024-11-28T03:41:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T03:41:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T03:41:29,859 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72 2024-11-28T03:41:29,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741964_1140 (size=53) 2024-11-28T03:41:29,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:41:29,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing eebe95cbdb6239965b947cdbd88704ed, disabling compactions & flushes 2024-11-28T03:41:29,871 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:29,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:29,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. after waiting 0 ms 2024-11-28T03:41:29,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:29,871 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:29,871 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for eebe95cbdb6239965b947cdbd88704ed: Waiting for close lock at 1732765289871Disabling compacts and flushes for region at 1732765289871Disabling writes for close at 1732765289871Writing region close event to WAL at 1732765289871Closed at 1732765289871 2024-11-28T03:41:29,872 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:41:29,873 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732765289873"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732765289873"}]},"ts":"1732765289873"} 2024-11-28T03:41:29,875 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:41:29,876 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:41:29,876 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765289876"}]},"ts":"1732765289876"} 2024-11-28T03:41:29,878 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T03:41:29,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, ASSIGN}] 2024-11-28T03:41:29,880 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, ASSIGN 2024-11-28T03:41:29,883 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, ASSIGN; state=OFFLINE, location=3a546af21488,34671,1732765249447; forceNewPlan=false, retain=false 2024-11-28T03:41:30,033 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=eebe95cbdb6239965b947cdbd88704ed, regionState=OPENING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:41:30,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, ASSIGN because future has completed 2024-11-28T03:41:30,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:41:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T03:41:30,193 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:30,193 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:41:30,193 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,193 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:41:30,194 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,194 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,205 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,207 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:41:30,208 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eebe95cbdb6239965b947cdbd88704ed columnFamilyName A 2024-11-28T03:41:30,209 DEBUG [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:30,209 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(327): Store=eebe95cbdb6239965b947cdbd88704ed/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:41:30,210 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,211 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:41:30,212 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eebe95cbdb6239965b947cdbd88704ed columnFamilyName B 2024-11-28T03:41:30,212 DEBUG [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:30,212 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(327): Store=eebe95cbdb6239965b947cdbd88704ed/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:41:30,213 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,214 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:41:30,214 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eebe95cbdb6239965b947cdbd88704ed columnFamilyName C 2024-11-28T03:41:30,214 DEBUG [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:30,215 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(327): Store=eebe95cbdb6239965b947cdbd88704ed/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:41:30,215 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:30,215 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,216 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,216 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,218 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,218 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,218 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:41:30,220 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,223 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:41:30,223 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened eebe95cbdb6239965b947cdbd88704ed; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70265610, jitterRate=0.047039180994033813}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:41:30,223 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:30,224 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for eebe95cbdb6239965b947cdbd88704ed: Running coprocessor pre-open hook at 1732765290194Writing region info on filesystem at 1732765290194Initializing all the Stores at 1732765290204 (+10 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765290204Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765290205 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765290205Cleaning up temporary data from old regions at 1732765290218 (+13 ms)Running coprocessor post-open hooks at 1732765290223 (+5 ms)Region opened successfully at 1732765290224 (+1 ms) 2024-11-28T03:41:30,228 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., pid=34, masterSystemTime=1732765290189 2024-11-28T03:41:30,233 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:30,233 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:30,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=eebe95cbdb6239965b947cdbd88704ed, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:41:30,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:41:30,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-28T03:41:30,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 in 202 msec 2024-11-28T03:41:30,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-28T03:41:30,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, ASSIGN in 361 msec 2024-11-28T03:41:30,244 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:41:30,244 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765290244"}]},"ts":"1732765290244"} 2024-11-28T03:41:30,247 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T03:41:30,248 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:41:30,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 810 msec 2024-11-28T03:41:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T03:41:30,577 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:30,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2739b942, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:30,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:30,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:30,580 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a24dae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:30,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31592bb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:30,584 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:30,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:30,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:30,588 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T03:41:30,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 3a546af21488,38327,1732765248628 2024-11-28T03:41:30,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b4f1196 2024-11-28T03:41:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T03:41:30,591 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T03:41:30,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T03:41:30,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:41:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:30,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741965_1141 (size=1097) 2024-11-28T03:41:31,028 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T03:41:31,028 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T03:41:31,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T03:41:31,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, REOPEN/MOVE}] 2024-11-28T03:41:31,040 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, REOPEN/MOVE 2024-11-28T03:41:31,040 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=eebe95cbdb6239965b947cdbd88704ed, regionState=CLOSING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:41:31,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, REOPEN/MOVE because future has completed 2024-11-28T03:41:31,043 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T03:41:31,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:41:31,196 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,196 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T03:41:31,196 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing eebe95cbdb6239965b947cdbd88704ed, disabling compactions & flushes 2024-11-28T03:41:31,196 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:31,196 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:31,196 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. after waiting 0 ms 2024-11-28T03:41:31,196 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:31,200 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T03:41:31,201 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:31,201 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for eebe95cbdb6239965b947cdbd88704ed: Waiting for close lock at 1732765291196Running coprocessor pre-close hooks at 1732765291196Disabling compacts and flushes for region at 1732765291196Disabling writes for close at 1732765291196Writing region close event to WAL at 1732765291197 (+1 ms)Running coprocessor post-close hooks at 1732765291201 (+4 ms)Closed at 1732765291201 2024-11-28T03:41:31,201 WARN [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: eebe95cbdb6239965b947cdbd88704ed to self. 2024-11-28T03:41:31,204 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,204 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=eebe95cbdb6239965b947cdbd88704ed, regionState=CLOSED 2024-11-28T03:41:31,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:41:31,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-28T03:41:31,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 in 165 msec 2024-11-28T03:41:31,211 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, REOPEN/MOVE; state=CLOSED, location=3a546af21488,34671,1732765249447; forceNewPlan=false, retain=true 2024-11-28T03:41:31,362 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=eebe95cbdb6239965b947cdbd88704ed, regionState=OPENING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:41:31,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, REOPEN/MOVE because future has completed 2024-11-28T03:41:31,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:41:31,525 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:31,525 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:41:31,525 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,525 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:41:31,526 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,526 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,537 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,538 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:41:31,546 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eebe95cbdb6239965b947cdbd88704ed columnFamilyName A 2024-11-28T03:41:31,549 DEBUG [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:31,551 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(327): Store=eebe95cbdb6239965b947cdbd88704ed/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:41:31,552 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,558 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:41:31,559 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eebe95cbdb6239965b947cdbd88704ed columnFamilyName B 2024-11-28T03:41:31,559 DEBUG [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:31,560 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(327): Store=eebe95cbdb6239965b947cdbd88704ed/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:41:31,560 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,562 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:41:31,565 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eebe95cbdb6239965b947cdbd88704ed columnFamilyName C 2024-11-28T03:41:31,565 DEBUG [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:31,566 INFO [StoreOpener-eebe95cbdb6239965b947cdbd88704ed-1 {}] regionserver.HStore(327): Store=eebe95cbdb6239965b947cdbd88704ed/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:41:31,566 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:31,566 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,567 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,569 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,570 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,570 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,571 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:41:31,572 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,574 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened eebe95cbdb6239965b947cdbd88704ed; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67597158, jitterRate=0.0072761476039886475}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:41:31,575 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,577 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for eebe95cbdb6239965b947cdbd88704ed: Running coprocessor pre-open hook at 1732765291526Writing region info on filesystem at 1732765291526Initializing all the Stores at 1732765291528 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765291528Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765291535 (+7 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765291535Cleaning up temporary data from old regions at 1732765291570 (+35 ms)Running coprocessor post-open hooks at 1732765291575 (+5 ms)Region opened successfully at 1732765291577 (+2 ms) 2024-11-28T03:41:31,579 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., pid=39, masterSystemTime=1732765291520 2024-11-28T03:41:31,581 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:31,581 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:31,582 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=eebe95cbdb6239965b947cdbd88704ed, regionState=OPEN, openSeqNum=5, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:41:31,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:41:31,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-28T03:41:31,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 in 221 msec 2024-11-28T03:41:31,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-28T03:41:31,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, REOPEN/MOVE in 550 msec 2024-11-28T03:41:31,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-28T03:41:31,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 563 msec 2024-11-28T03:41:31,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 998 msec 2024-11-28T03:41:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-28T03:41:31,610 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:31,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60bc1163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:31,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:31,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fdf8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:31,625 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db77b9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695fea61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:31,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:31,642 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e49d746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:31,644 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:31,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b6fb02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b90f27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:31,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:31,657 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ccac660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:31,660 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@566f0f03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75252226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:31,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:31,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@266d5f4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:31,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:31,672 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab7e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66218f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:31,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:31,682 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e86a95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:31,686 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f1a643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d8bca89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:31,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:31,689 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ed5007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:31,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ea9e70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2e194a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:31,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:31,697 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f8ade5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:31,699 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@410e6ba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1033cef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,703 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:31,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:31,704 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3406cbf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:31,707 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40765e5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f82fe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:31,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:31,715 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a3720be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:31,719 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b03e75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:31,728 DEBUG [Thread-715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:31,733 DEBUG [Thread-706 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:31,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:31,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:31,736 DEBUG [Thread-698 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:31,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:41:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:31,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:41:31,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:31,742 DEBUG [Thread-702 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:31,743 DEBUG [Thread-700 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:41:31,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:31,750 DEBUG [Thread-713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:31,752 DEBUG [Thread-704 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:41:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:31,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:41:31,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:31,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:41:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T03:41:31,765 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-28T03:41:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T03:41:31,767 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:31,768 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:31,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:41:31,780 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:31,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:31,785 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:31,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:41:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:31,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:31,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:41:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T03:41:31,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128af88d97ec0fe4ed2b1ac3cf5c0b8a995_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765291763/Put/seqid=0 2024-11-28T03:41:31,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:31,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765351889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:31,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765351892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:31,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765351892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:31,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765351896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:31,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765351898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:31,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T03:41:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:31,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:31,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741966_1142 (size=14594) 2024-11-28T03:41:31,942 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:31,955 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128af88d97ec0fe4ed2b1ac3cf5c0b8a995_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128af88d97ec0fe4ed2b1ac3cf5c0b8a995_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:31,960 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/d9de935f02d04ecdbe919e46105b41cb, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:31,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/d9de935f02d04ecdbe919e46105b41cb is 175, key is test_row_0/A:col10/1732765291763/Put/seqid=0 2024-11-28T03:41:31,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741967_1143 (size=39549) 2024-11-28T03:41:32,002 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/d9de935f02d04ecdbe919e46105b41cb 2024-11-28T03:41:32,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765352017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765352017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765352024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765352024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765352027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/b24f188b840949ee9e34ead8c6a8b4b0 is 50, key is test_row_0/B:col10/1732765291763/Put/seqid=0 2024-11-28T03:41:32,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741968_1144 (size=12001) 2024-11-28T03:41:32,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T03:41:32,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:32,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:32,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:32,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T03:41:32,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765352227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T03:41:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:32,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:32,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:32,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765352227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765352236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765352237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765352236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T03:41:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:32,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T03:41:32,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/b24f188b840949ee9e34ead8c6a8b4b0 2024-11-28T03:41:32,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/6b1aebbabf6e47cfad03c875f3be9cb0 is 50, key is test_row_0/C:col10/1732765291763/Put/seqid=0 2024-11-28T03:41:32,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T03:41:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:32,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:32,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:32,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:32,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765352536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765352547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765352556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765352556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765352557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:32,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741969_1145 (size=12001) 2024-11-28T03:41:32,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/6b1aebbabf6e47cfad03c875f3be9cb0 2024-11-28T03:41:32,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/d9de935f02d04ecdbe919e46105b41cb as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/d9de935f02d04ecdbe919e46105b41cb 2024-11-28T03:41:32,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/d9de935f02d04ecdbe919e46105b41cb, entries=200, sequenceid=17, filesize=38.6 K 2024-11-28T03:41:32,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/b24f188b840949ee9e34ead8c6a8b4b0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/b24f188b840949ee9e34ead8c6a8b4b0 2024-11-28T03:41:32,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/b24f188b840949ee9e34ead8c6a8b4b0, entries=150, sequenceid=17, filesize=11.7 K 2024-11-28T03:41:32,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/6b1aebbabf6e47cfad03c875f3be9cb0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6b1aebbabf6e47cfad03c875f3be9cb0 2024-11-28T03:41:32,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6b1aebbabf6e47cfad03c875f3be9cb0, entries=150, sequenceid=17, filesize=11.7 K 2024-11-28T03:41:32,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for eebe95cbdb6239965b947cdbd88704ed in 868ms, sequenceid=17, compaction requested=false 2024-11-28T03:41:32,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:32,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T03:41:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:32,696 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:41:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:32,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:32,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112803d284421ae7479ab7a6f489da52ca97_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765291891/Put/seqid=0 2024-11-28T03:41:32,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741970_1146 (size=12154) 2024-11-28T03:41:32,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:32,814 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112803d284421ae7479ab7a6f489da52ca97_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112803d284421ae7479ab7a6f489da52ca97_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/71171ad6e007428895385b66fc6f38f2, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/71171ad6e007428895385b66fc6f38f2 is 175, key is test_row_0/A:col10/1732765291891/Put/seqid=0 2024-11-28T03:41:32,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741971_1147 (size=30955) 2024-11-28T03:41:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T03:41:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:33,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:33,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765353119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765353127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765353129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765353133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765353133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765353236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765353236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765353247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765353247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765353247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,251 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/71171ad6e007428895385b66fc6f38f2 2024-11-28T03:41:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/e3240e83a13e446ebb5163dc731a1c3e is 50, key is test_row_0/B:col10/1732765291891/Put/seqid=0 2024-11-28T03:41:33,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741972_1148 (size=12001) 2024-11-28T03:41:33,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765353446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765353446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765353456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765353456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765353457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,680 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/e3240e83a13e446ebb5163dc731a1c3e 2024-11-28T03:41:33,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/50c5401f619f42eda77316d31339b192 is 50, key is test_row_0/C:col10/1732765291891/Put/seqid=0 2024-11-28T03:41:33,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741973_1149 (size=12001) 2024-11-28T03:41:33,751 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/50c5401f619f42eda77316d31339b192 2024-11-28T03:41:33,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/71171ad6e007428895385b66fc6f38f2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/71171ad6e007428895385b66fc6f38f2 2024-11-28T03:41:33,760 INFO [master/3a546af21488:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-28T03:41:33,760 INFO [master/3a546af21488:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-28T03:41:33,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765353758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765353757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,770 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/71171ad6e007428895385b66fc6f38f2, entries=150, sequenceid=36, filesize=30.2 K 2024-11-28T03:41:33,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765353769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/e3240e83a13e446ebb5163dc731a1c3e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e3240e83a13e446ebb5163dc731a1c3e 2024-11-28T03:41:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,779 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e3240e83a13e446ebb5163dc731a1c3e, entries=150, sequenceid=36, filesize=11.7 K 2024-11-28T03:41:33,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765353776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765353776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:33,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/50c5401f619f42eda77316d31339b192 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/50c5401f619f42eda77316d31339b192 2024-11-28T03:41:33,788 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/50c5401f619f42eda77316d31339b192, entries=150, sequenceid=36, filesize=11.7 K 2024-11-28T03:41:33,790 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for eebe95cbdb6239965b947cdbd88704ed in 1093ms, sequenceid=36, compaction requested=false 2024-11-28T03:41:33,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:33,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:33,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-28T03:41:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-28T03:41:33,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-28T03:41:33,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0240 sec 2024-11-28T03:41:33,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.0330 sec 2024-11-28T03:41:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T03:41:33,917 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:33,918 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-28T03:41:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T03:41:33,923 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:33,923 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:33,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T03:41:34,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T03:41:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:34,078 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T03:41:34,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:34,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:34,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:34,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:34,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:34,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c1162b9acedb462e9d98423bd340a1c7_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765293129/Put/seqid=0 2024-11-28T03:41:34,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741974_1150 (size=12154) 2024-11-28T03:41:34,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:34,116 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c1162b9acedb462e9d98423bd340a1c7_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c1162b9acedb462e9d98423bd340a1c7_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:34,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/0aea09312f9d4c158e38322d58280f0b, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:34,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/0aea09312f9d4c158e38322d58280f0b is 175, key is test_row_0/A:col10/1732765293129/Put/seqid=0 2024-11-28T03:41:34,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741975_1151 (size=30955) 2024-11-28T03:41:34,123 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/0aea09312f9d4c158e38322d58280f0b 2024-11-28T03:41:34,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/c409f570180c400c81fd77ad27854da7 is 50, key is test_row_0/B:col10/1732765293129/Put/seqid=0 2024-11-28T03:41:34,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741976_1152 (size=12001) 2024-11-28T03:41:34,139 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/c409f570180c400c81fd77ad27854da7 2024-11-28T03:41:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/fade6c51120c48d39a34b5be7b73b6d9 is 50, key is test_row_0/C:col10/1732765293129/Put/seqid=0 2024-11-28T03:41:34,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741977_1153 (size=12001) 2024-11-28T03:41:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T03:41:34,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:34,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:34,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765354299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,308 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:41:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765354301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765354303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765354304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765354310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765354417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765354417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765354417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765354418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765354426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T03:41:34,556 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/fade6c51120c48d39a34b5be7b73b6d9 2024-11-28T03:41:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/0aea09312f9d4c158e38322d58280f0b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0aea09312f9d4c158e38322d58280f0b 2024-11-28T03:41:34,569 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0aea09312f9d4c158e38322d58280f0b, entries=150, sequenceid=49, filesize=30.2 K 2024-11-28T03:41:34,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/c409f570180c400c81fd77ad27854da7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c409f570180c400c81fd77ad27854da7 2024-11-28T03:41:34,579 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c409f570180c400c81fd77ad27854da7, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T03:41:34,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/fade6c51120c48d39a34b5be7b73b6d9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fade6c51120c48d39a34b5be7b73b6d9 2024-11-28T03:41:34,590 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fade6c51120c48d39a34b5be7b73b6d9, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T03:41:34,591 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for eebe95cbdb6239965b947cdbd88704ed in 513ms, sequenceid=49, compaction requested=true 2024-11-28T03:41:34,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:34,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:34,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-28T03:41:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-28T03:41:34,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-28T03:41:34,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 669 msec 2024-11-28T03:41:34,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 677 msec 2024-11-28T03:41:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:34,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T03:41:34,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765294292, max timestamp=1732765294630Num uniques 859; 2024-11-28T03:41:34,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765294292, max timestamp=1732765294630Num uniques 859; 2024-11-28T03:41:34,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:41:34,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:41:34,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765294292, max timestamp=1732765294630Num uniques 859; 2024-11-28T03:41:34,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:41:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:34,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ee43c593418b4df291510bd65a07c6b4_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765294305/Put/seqid=0 2024-11-28T03:41:34,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741978_1154 (size=12154) 2024-11-28T03:41:34,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765294674, max timestamp=1732765294711Num uniques 859; 2024-11-28T03:41:34,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:41:34,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765294674, max timestamp=1732765294711Num uniques 859; 2024-11-28T03:41:34,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:41:34,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765294674, max timestamp=1732765294711Num uniques 859; 2024-11-28T03:41:34,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:41:34,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732765294676, max timestamp=1732765294756Num uniques 1118; 2024-11-28T03:41:34,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732765294676, max timestamp=1732765294756Num uniques 1118; 2024-11-28T03:41:34,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-11-28T03:41:34,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-11-28T03:41:34,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732765294676, max timestamp=1732765294756Num uniques 1118; 2024-11-28T03:41:34,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-11-28T03:41:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765354800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765354800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765354802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765354802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765354804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765354917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765354917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765354918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765354918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:34,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765354917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T03:41:35,057 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:35,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-28T03:41:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T03:41:35,060 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:35,061 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:35,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:35,084 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:35,095 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ee43c593418b4df291510bd65a07c6b4_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ee43c593418b4df291510bd65a07c6b4_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:35,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/c8f1b579f1674894b7d16efe794d6862, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:35,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/c8f1b579f1674894b7d16efe794d6862 is 175, key is test_row_0/A:col10/1732765294305/Put/seqid=0 2024-11-28T03:41:35,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741979_1155 (size=30955) 2024-11-28T03:41:35,124 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/c8f1b579f1674894b7d16efe794d6862 2024-11-28T03:41:35,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765355126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765355127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765355127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765355127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765355136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/c784bcaeaa424afb96dc505d4b1f9b39 is 50, key is test_row_0/B:col10/1732765294305/Put/seqid=0 2024-11-28T03:41:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T03:41:35,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741980_1156 (size=12001) 2024-11-28T03:41:35,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T03:41:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:35,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:35,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T03:41:35,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:35,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:35,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:35,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T03:41:35,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765355447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765355447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765355447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765355448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765355449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T03:41:35,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:35,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:35,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:35,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:35,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:35,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/c784bcaeaa424afb96dc505d4b1f9b39 2024-11-28T03:41:35,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/dd20e2bf311243efa1a6b8f3106d7d10 is 50, key is test_row_0/C:col10/1732765294305/Put/seqid=0 2024-11-28T03:41:35,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741981_1157 (size=12001) 2024-11-28T03:41:35,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/dd20e2bf311243efa1a6b8f3106d7d10 2024-11-28T03:41:35,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/c8f1b579f1674894b7d16efe794d6862 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c8f1b579f1674894b7d16efe794d6862 2024-11-28T03:41:35,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c8f1b579f1674894b7d16efe794d6862, entries=150, sequenceid=70, filesize=30.2 K 2024-11-28T03:41:35,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/c784bcaeaa424afb96dc505d4b1f9b39 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c784bcaeaa424afb96dc505d4b1f9b39 2024-11-28T03:41:35,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c784bcaeaa424afb96dc505d4b1f9b39, entries=150, sequenceid=70, filesize=11.7 K 2024-11-28T03:41:35,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/dd20e2bf311243efa1a6b8f3106d7d10 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/dd20e2bf311243efa1a6b8f3106d7d10 2024-11-28T03:41:35,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/dd20e2bf311243efa1a6b8f3106d7d10, entries=150, sequenceid=70, filesize=11.7 K 2024-11-28T03:41:35,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=161.02 KB/164880 for eebe95cbdb6239965b947cdbd88704ed in 1051ms, sequenceid=70, compaction requested=true 2024-11-28T03:41:35,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:35,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:35,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:35,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:35,682 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:35,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:35,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:35,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T03:41:35,682 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:35,684 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:35,684 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/A is initiating minor compaction (all files) 2024-11-28T03:41:35,684 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/A in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:35,684 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/d9de935f02d04ecdbe919e46105b41cb, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/71171ad6e007428895385b66fc6f38f2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0aea09312f9d4c158e38322d58280f0b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c8f1b579f1674894b7d16efe794d6862] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=129.3 K 2024-11-28T03:41:35,684 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:35,684 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/d9de935f02d04ecdbe919e46105b41cb, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/71171ad6e007428895385b66fc6f38f2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0aea09312f9d4c158e38322d58280f0b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c8f1b579f1674894b7d16efe794d6862] 2024-11-28T03:41:35,685 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:35,685 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/B is initiating minor compaction (all files) 2024-11-28T03:41:35,685 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/B in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:35,685 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/b24f188b840949ee9e34ead8c6a8b4b0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e3240e83a13e446ebb5163dc731a1c3e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c409f570180c400c81fd77ad27854da7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c784bcaeaa424afb96dc505d4b1f9b39] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=46.9 K 2024-11-28T03:41:35,686 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d9de935f02d04ecdbe919e46105b41cb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732765291754 2024-11-28T03:41:35,686 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b24f188b840949ee9e34ead8c6a8b4b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732765291754 2024-11-28T03:41:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T03:41:35,687 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 71171ad6e007428895385b66fc6f38f2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732765291867 2024-11-28T03:41:35,687 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3240e83a13e446ebb5163dc731a1c3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732765291867 2024-11-28T03:41:35,687 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aea09312f9d4c158e38322d58280f0b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732765293125 2024-11-28T03:41:35,688 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c409f570180c400c81fd77ad27854da7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732765293125 2024-11-28T03:41:35,688 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f1b579f1674894b7d16efe794d6862, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732765294292 2024-11-28T03:41:35,688 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c784bcaeaa424afb96dc505d4b1f9b39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732765294292 2024-11-28T03:41:35,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T03:41:35,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:35,692 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=161.02 KB heapSize=445.08 KB 2024-11-28T03:41:35,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:35,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:35,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:35,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:35,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:35,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:35,717 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:35,736 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#B#compaction#139 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:35,737 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/7d76df2e22274dd8a121e34fcf313b8c is 50, key is test_row_0/B:col10/1732765294305/Put/seqid=0 2024-11-28T03:41:35,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fe157ab495e8483ab865d3ea3d551136_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765294802/Put/seqid=0 2024-11-28T03:41:35,761 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411285187a7627cc341b4a51c87891223a970_eebe95cbdb6239965b947cdbd88704ed store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:35,768 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411285187a7627cc341b4a51c87891223a970_eebe95cbdb6239965b947cdbd88704ed, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:35,769 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285187a7627cc341b4a51c87891223a970_eebe95cbdb6239965b947cdbd88704ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:35,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741982_1158 (size=12139) 2024-11-28T03:41:35,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741984_1160 (size=4469) 2024-11-28T03:41:35,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741983_1159 (size=12154) 2024-11-28T03:41:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:35,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:35,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765355958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765355964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765355964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765355965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765355965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765356077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765356077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765356077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765356077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T03:41:36,233 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/7d76df2e22274dd8a121e34fcf313b8c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/7d76df2e22274dd8a121e34fcf313b8c 2024-11-28T03:41:36,242 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/B of eebe95cbdb6239965b947cdbd88704ed into 7d76df2e22274dd8a121e34fcf313b8c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:36,242 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:36,242 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/B, priority=12, startTime=1732765295682; duration=0sec 2024-11-28T03:41:36,242 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:36,242 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:B 2024-11-28T03:41:36,242 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:36,246 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:36,246 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/C is initiating minor compaction (all files) 2024-11-28T03:41:36,246 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/C in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:36,246 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6b1aebbabf6e47cfad03c875f3be9cb0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/50c5401f619f42eda77316d31339b192, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fade6c51120c48d39a34b5be7b73b6d9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/dd20e2bf311243efa1a6b8f3106d7d10] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=46.9 K 2024-11-28T03:41:36,247 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b1aebbabf6e47cfad03c875f3be9cb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732765291754 2024-11-28T03:41:36,247 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c5401f619f42eda77316d31339b192, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732765291867 2024-11-28T03:41:36,248 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fade6c51120c48d39a34b5be7b73b6d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732765293125 2024-11-28T03:41:36,248 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd20e2bf311243efa1a6b8f3106d7d10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732765294292 2024-11-28T03:41:36,267 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:36,268 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f7a4b8b5276c46c4b3b24988bec7efa8 is 50, key is test_row_0/C:col10/1732765294305/Put/seqid=0 2024-11-28T03:41:36,270 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#A#compaction#138 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:36,271 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/cca8c4ab9447434ea901b1fa8c3e353d is 175, key is test_row_0/A:col10/1732765294305/Put/seqid=0 2024-11-28T03:41:36,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:36,291 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fe157ab495e8483ab865d3ea3d551136_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fe157ab495e8483ab865d3ea3d551136_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:36,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765356287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/c302a6a09c9442188d1b024ae3d7b8c0, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:36,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/c302a6a09c9442188d1b024ae3d7b8c0 is 175, key is test_row_0/A:col10/1732765294802/Put/seqid=0 2024-11-28T03:41:36,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741985_1161 (size=12139) 2024-11-28T03:41:36,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765356287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765356287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741986_1162 (size=31093) 2024-11-28T03:41:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765356297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,312 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f7a4b8b5276c46c4b3b24988bec7efa8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f7a4b8b5276c46c4b3b24988bec7efa8 2024-11-28T03:41:36,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741987_1163 (size=30955) 2024-11-28T03:41:36,320 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/cca8c4ab9447434ea901b1fa8c3e353d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/cca8c4ab9447434ea901b1fa8c3e353d 2024-11-28T03:41:36,321 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=124, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/c302a6a09c9442188d1b024ae3d7b8c0 2024-11-28T03:41:36,329 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/C of eebe95cbdb6239965b947cdbd88704ed into f7a4b8b5276c46c4b3b24988bec7efa8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:36,329 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:36,329 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/C, priority=12, startTime=1732765295682; duration=0sec 2024-11-28T03:41:36,329 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:36,329 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:C 2024-11-28T03:41:36,333 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/A of eebe95cbdb6239965b947cdbd88704ed into cca8c4ab9447434ea901b1fa8c3e353d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:36,333 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:36,333 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/A, priority=12, startTime=1732765295681; duration=0sec 2024-11-28T03:41:36,333 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:36,333 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:A 2024-11-28T03:41:36,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/8cf4dd48ea594e4f85ad3de28e1e7427 is 50, key is test_row_0/B:col10/1732765294802/Put/seqid=0 2024-11-28T03:41:36,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741988_1164 (size=12001) 2024-11-28T03:41:36,378 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/8cf4dd48ea594e4f85ad3de28e1e7427 2024-11-28T03:41:36,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4f667817fd444ea4801e33f9813a9de9 is 50, key is test_row_0/C:col10/1732765294802/Put/seqid=0 2024-11-28T03:41:36,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741989_1165 (size=12001) 2024-11-28T03:41:36,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765356607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765356607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765356607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765356608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:36,832 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4f667817fd444ea4801e33f9813a9de9 2024-11-28T03:41:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/c302a6a09c9442188d1b024ae3d7b8c0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c302a6a09c9442188d1b024ae3d7b8c0 2024-11-28T03:41:36,846 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c302a6a09c9442188d1b024ae3d7b8c0, entries=150, sequenceid=124, filesize=30.2 K 2024-11-28T03:41:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/8cf4dd48ea594e4f85ad3de28e1e7427 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8cf4dd48ea594e4f85ad3de28e1e7427 2024-11-28T03:41:36,855 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8cf4dd48ea594e4f85ad3de28e1e7427, entries=150, sequenceid=124, filesize=11.7 K 2024-11-28T03:41:36,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4f667817fd444ea4801e33f9813a9de9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f667817fd444ea4801e33f9813a9de9 2024-11-28T03:41:36,861 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f667817fd444ea4801e33f9813a9de9, entries=150, sequenceid=124, filesize=11.7 K 2024-11-28T03:41:36,863 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.03 KB/455712, currentSize=26.84 KB/27480 for eebe95cbdb6239965b947cdbd88704ed in 1171ms, sequenceid=124, compaction requested=false 2024-11-28T03:41:36,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:36,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:36,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-28T03:41:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-28T03:41:36,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-28T03:41:36,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8040 sec 2024-11-28T03:41:36,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8110 sec 2024-11-28T03:41:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:37,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:41:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:37,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fdbbba6108064509889ab49ff5d3d924_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765296989/Put/seqid=0 2024-11-28T03:41:37,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741990_1166 (size=12254) 2024-11-28T03:41:37,066 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:37,071 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fdbbba6108064509889ab49ff5d3d924_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fdbbba6108064509889ab49ff5d3d924_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:37,072 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/43f8a65f64f844199c43201f5454faf2, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:37,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/43f8a65f64f844199c43201f5454faf2 is 175, key is test_row_0/A:col10/1732765296989/Put/seqid=0 2024-11-28T03:41:37,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741991_1167 (size=31055) 2024-11-28T03:41:37,098 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/43f8a65f64f844199c43201f5454faf2 2024-11-28T03:41:37,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/e92a205406a24b96bac8dda6ffcc8feb is 50, key is test_row_0/B:col10/1732765296989/Put/seqid=0 2024-11-28T03:41:37,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741992_1168 (size=12101) 2024-11-28T03:41:37,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/e92a205406a24b96bac8dda6ffcc8feb 2024-11-28T03:41:37,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732765297041, max timestamp=1732765297123Num uniques 859; 2024-11-28T03:41:37,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732765297041, max timestamp=1732765297123Num uniques 859; 2024-11-28T03:41:37,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732765297041, max timestamp=1732765297123Num uniques 859; 2024-11-28T03:41:37,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T03:41:37,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T03:41:37,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T03:41:37,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/eaa25eec4e244225bb3bb7fa1f4c4bec is 50, key is test_row_0/C:col10/1732765296989/Put/seqid=0 2024-11-28T03:41:37,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741993_1169 (size=12101) 2024-11-28T03:41:37,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765357147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765357152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765357157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765357159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765357159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T03:41:37,207 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:37,207 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-28T03:41:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T03:41:37,210 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:37,211 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:37,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:37,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765357276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765357277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765357277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765357277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765357277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T03:41:37,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T03:41:37,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:37,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:37,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765357487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765357487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765357487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765357487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765357488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T03:41:37,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:37,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:37,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:37,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:37,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T03:41:37,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/eaa25eec4e244225bb3bb7fa1f4c4bec 2024-11-28T03:41:37,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/43f8a65f64f844199c43201f5454faf2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/43f8a65f64f844199c43201f5454faf2 2024-11-28T03:41:37,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/43f8a65f64f844199c43201f5454faf2, entries=150, sequenceid=137, filesize=30.3 K 2024-11-28T03:41:37,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/e92a205406a24b96bac8dda6ffcc8feb as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e92a205406a24b96bac8dda6ffcc8feb 2024-11-28T03:41:37,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e92a205406a24b96bac8dda6ffcc8feb, entries=150, sequenceid=137, filesize=11.8 K 2024-11-28T03:41:37,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/eaa25eec4e244225bb3bb7fa1f4c4bec as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/eaa25eec4e244225bb3bb7fa1f4c4bec 2024-11-28T03:41:37,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/eaa25eec4e244225bb3bb7fa1f4c4bec, entries=150, sequenceid=137, filesize=11.8 K 2024-11-28T03:41:37,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for eebe95cbdb6239965b947cdbd88704ed in 576ms, sequenceid=137, compaction requested=true 2024-11-28T03:41:37,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:37,576 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:37,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:37,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:37,577 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:37,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:37,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:37,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:37,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:37,580 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93103 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:37,580 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/A is initiating minor compaction (all files) 2024-11-28T03:41:37,580 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/A in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:37,580 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/cca8c4ab9447434ea901b1fa8c3e353d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c302a6a09c9442188d1b024ae3d7b8c0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/43f8a65f64f844199c43201f5454faf2] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=90.9 K 2024-11-28T03:41:37,580 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:37,580 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/cca8c4ab9447434ea901b1fa8c3e353d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c302a6a09c9442188d1b024ae3d7b8c0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/43f8a65f64f844199c43201f5454faf2] 2024-11-28T03:41:37,581 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:37,581 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/B is initiating minor compaction (all files) 2024-11-28T03:41:37,581 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/B in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:37,581 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/7d76df2e22274dd8a121e34fcf313b8c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8cf4dd48ea594e4f85ad3de28e1e7427, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e92a205406a24b96bac8dda6ffcc8feb] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=35.4 K 2024-11-28T03:41:37,582 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cca8c4ab9447434ea901b1fa8c3e353d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732765294292 2024-11-28T03:41:37,582 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d76df2e22274dd8a121e34fcf313b8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732765294292 2024-11-28T03:41:37,582 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c302a6a09c9442188d1b024ae3d7b8c0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732765294768 2024-11-28T03:41:37,582 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cf4dd48ea594e4f85ad3de28e1e7427, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732765294768 2024-11-28T03:41:37,583 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43f8a65f64f844199c43201f5454faf2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732765295959 2024-11-28T03:41:37,583 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e92a205406a24b96bac8dda6ffcc8feb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732765295959 2024-11-28T03:41:37,593 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:37,593 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/d3d52da06fdf4dc29992eb6bdbf5f19a is 50, key is test_row_0/B:col10/1732765296989/Put/seqid=0 2024-11-28T03:41:37,601 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:37,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741994_1170 (size=12341) 2024-11-28T03:41:37,604 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128a0a2dacdd36b4a11b394deed887e379a_eebe95cbdb6239965b947cdbd88704ed store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:37,607 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128a0a2dacdd36b4a11b394deed887e379a_eebe95cbdb6239965b947cdbd88704ed, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:37,607 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a0a2dacdd36b4a11b394deed887e379a_eebe95cbdb6239965b947cdbd88704ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:37,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741995_1171 (size=4469) 2024-11-28T03:41:37,618 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/d3d52da06fdf4dc29992eb6bdbf5f19a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/d3d52da06fdf4dc29992eb6bdbf5f19a 2024-11-28T03:41:37,627 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/B of eebe95cbdb6239965b947cdbd88704ed into d3d52da06fdf4dc29992eb6bdbf5f19a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:37,627 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:37,627 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/B, priority=13, startTime=1732765297577; duration=0sec 2024-11-28T03:41:37,627 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:37,627 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:B 2024-11-28T03:41:37,627 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:37,629 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:37,629 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/C is initiating minor compaction (all files) 2024-11-28T03:41:37,629 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/C in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:37,629 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f7a4b8b5276c46c4b3b24988bec7efa8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f667817fd444ea4801e33f9813a9de9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/eaa25eec4e244225bb3bb7fa1f4c4bec] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=35.4 K 2024-11-28T03:41:37,630 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting f7a4b8b5276c46c4b3b24988bec7efa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732765294292 2024-11-28T03:41:37,631 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f667817fd444ea4801e33f9813a9de9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732765294768 2024-11-28T03:41:37,631 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa25eec4e244225bb3bb7fa1f4c4bec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732765295959 2024-11-28T03:41:37,642 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#C#compaction#149 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:37,642 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/07b8561270f54e43a24085f3589bc099 is 50, key is test_row_0/C:col10/1732765296989/Put/seqid=0 2024-11-28T03:41:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741996_1172 (size=12341) 2024-11-28T03:41:37,667 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/07b8561270f54e43a24085f3589bc099 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/07b8561270f54e43a24085f3589bc099 2024-11-28T03:41:37,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T03:41:37,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:37,672 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-28T03:41:37,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:37,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:37,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:37,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:37,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:37,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:37,676 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/C of eebe95cbdb6239965b947cdbd88704ed into 07b8561270f54e43a24085f3589bc099(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:37,676 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:37,676 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/C, priority=13, startTime=1732765297577; duration=0sec 2024-11-28T03:41:37,677 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:37,677 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:C 2024-11-28T03:41:37,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128106441e3c43a4ed38d739ec192cc566a_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765297157/Put/seqid=0 2024-11-28T03:41:37,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741997_1173 (size=12304) 2024-11-28T03:41:37,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:37,722 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128106441e3c43a4ed38d739ec192cc566a_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128106441e3c43a4ed38d739ec192cc566a_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:37,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/1a619ed78a254c0fa1ebf9397bee02c4, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:37,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/1a619ed78a254c0fa1ebf9397bee02c4 is 175, key is test_row_0/A:col10/1732765297157/Put/seqid=0 2024-11-28T03:41:37,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741998_1174 (size=31105) 2024-11-28T03:41:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:37,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T03:41:37,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765357848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765357848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765357849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765357850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765357850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765357966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765357966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765357967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765357967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765357967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,019 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#A#compaction#148 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:38,020 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/b4d3822a7e17428384dc1d1fe914dd97 is 175, key is test_row_0/A:col10/1732765296989/Put/seqid=0 2024-11-28T03:41:38,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741999_1175 (size=31295) 2024-11-28T03:41:38,144 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/1a619ed78a254c0fa1ebf9397bee02c4 2024-11-28T03:41:38,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/2054df253e684f74a836184be700c3df is 50, key is test_row_0/B:col10/1732765297157/Put/seqid=0 2024-11-28T03:41:38,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742000_1176 (size=12151) 2024-11-28T03:41:38,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765358176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:38,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:38,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765358177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:38,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765358177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765358177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765358177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T03:41:38,430 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/b4d3822a7e17428384dc1d1fe914dd97 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/b4d3822a7e17428384dc1d1fe914dd97 2024-11-28T03:41:38,438 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/A of eebe95cbdb6239965b947cdbd88704ed into b4d3822a7e17428384dc1d1fe914dd97(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:38,438 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:38,438 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/A, priority=13, startTime=1732765297576; duration=0sec 2024-11-28T03:41:38,438 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:38,438 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:A 2024-11-28T03:41:38,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765358489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:38,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765358489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:38,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765358489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765358489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765358489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:38,571 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/2054df253e684f74a836184be700c3df 2024-11-28T03:41:38,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/5fdba96fec8640c79f9236e96a6db50a is 50, key is test_row_0/C:col10/1732765297157/Put/seqid=0 2024-11-28T03:41:38,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742001_1177 (size=12151) 2024-11-28T03:41:38,998 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/5fdba96fec8640c79f9236e96a6db50a 2024-11-28T03:41:39,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765358996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:39,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765358997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:39,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765358997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:39,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765358999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/1a619ed78a254c0fa1ebf9397bee02c4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/1a619ed78a254c0fa1ebf9397bee02c4 2024-11-28T03:41:39,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765359006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:39,015 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/1a619ed78a254c0fa1ebf9397bee02c4, entries=150, sequenceid=172, filesize=30.4 K 2024-11-28T03:41:39,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/2054df253e684f74a836184be700c3df as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/2054df253e684f74a836184be700c3df 2024-11-28T03:41:39,026 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/2054df253e684f74a836184be700c3df, entries=150, sequenceid=172, filesize=11.9 K 2024-11-28T03:41:39,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/5fdba96fec8640c79f9236e96a6db50a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/5fdba96fec8640c79f9236e96a6db50a 2024-11-28T03:41:39,034 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/5fdba96fec8640c79f9236e96a6db50a, entries=150, sequenceid=172, filesize=11.9 K 2024-11-28T03:41:39,035 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=53.67 KB/54960 for eebe95cbdb6239965b947cdbd88704ed in 1363ms, sequenceid=172, compaction requested=false 2024-11-28T03:41:39,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:39,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:39,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-28T03:41:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-28T03:41:39,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-28T03:41:39,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8280 sec 2024-11-28T03:41:39,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8350 sec 2024-11-28T03:41:39,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T03:41:39,078 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T03:41:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T03:41:39,357 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:39,357 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-28T03:41:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T03:41:39,359 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:39,360 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:39,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T03:41:39,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T03:41:39,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:39,532 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T03:41:39,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:39,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:39,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:39,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:39,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:39,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:39,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128490908877d594d488e4ebcbd7f3be149_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765297847/Put/seqid=0 2024-11-28T03:41:39,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742002_1178 (size=12304) 2024-11-28T03:41:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T03:41:39,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:39,968 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128490908877d594d488e4ebcbd7f3be149_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128490908877d594d488e4ebcbd7f3be149_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:39,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/fe3388b1b7ec42f084539291557101fd, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:39,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/fe3388b1b7ec42f084539291557101fd is 175, key is test_row_0/A:col10/1732765297847/Put/seqid=0 2024-11-28T03:41:39,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742003_1179 (size=31105) 2024-11-28T03:41:39,985 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/fe3388b1b7ec42f084539291557101fd 2024-11-28T03:41:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T03:41:39,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/17fa0eff0bc0410ab533b299f0709d68 is 50, key is test_row_0/B:col10/1732765297847/Put/seqid=0 2024-11-28T03:41:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:40,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742004_1180 (size=12151) 2024-11-28T03:41:40,035 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/17fa0eff0bc0410ab533b299f0709d68 2024-11-28T03:41:40,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/890ebb910f9d4a489346eba136ace9f6 is 50, key is test_row_0/C:col10/1732765297847/Put/seqid=0 2024-11-28T03:41:40,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742005_1181 (size=12151) 2024-11-28T03:41:40,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765300044, max timestamp=1732765300063Num uniques 859; 2024-11-28T03:41:40,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:41:40,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765300044, max timestamp=1732765300063Num uniques 859; 2024-11-28T03:41:40,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:41:40,068 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/890ebb910f9d4a489346eba136ace9f6 2024-11-28T03:41:40,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765300044, max timestamp=1732765300063Num uniques 859; 2024-11-28T03:41:40,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:41:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/fe3388b1b7ec42f084539291557101fd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/fe3388b1b7ec42f084539291557101fd 2024-11-28T03:41:40,088 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/fe3388b1b7ec42f084539291557101fd, entries=150, sequenceid=185, filesize=30.4 K 2024-11-28T03:41:40,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/17fa0eff0bc0410ab533b299f0709d68 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/17fa0eff0bc0410ab533b299f0709d68 2024-11-28T03:41:40,101 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/17fa0eff0bc0410ab533b299f0709d68, entries=150, sequenceid=185, filesize=11.9 K 2024-11-28T03:41:40,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/890ebb910f9d4a489346eba136ace9f6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/890ebb910f9d4a489346eba136ace9f6 2024-11-28T03:41:40,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765360097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765360099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,109 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/890ebb910f9d4a489346eba136ace9f6, entries=150, sequenceid=185, filesize=11.9 K 2024-11-28T03:41:40,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765360104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,110 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for eebe95cbdb6239965b947cdbd88704ed in 579ms, sequenceid=185, compaction requested=true 2024-11-28T03:41:40,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:40,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:40,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-28T03:41:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-28T03:41:40,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-28T03:41:40,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 752 msec 2024-11-28T03:41:40,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 759 msec 2024-11-28T03:41:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:40,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-28T03:41:40,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:40,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:40,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:40,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:40,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:40,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:40,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765360151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128de54851322c646798ec8dae6b47be647_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765300121/Put/seqid=0 2024-11-28T03:41:40,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742006_1182 (size=14794) 2024-11-28T03:41:40,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765360159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765360216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765360217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765360217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765360266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765360276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765360426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765360427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765360427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765360477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T03:41:40,496 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:40,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-28T03:41:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T03:41:40,500 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:40,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765360496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,501 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:40,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:40,569 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:40,578 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128de54851322c646798ec8dae6b47be647_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128de54851322c646798ec8dae6b47be647_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:40,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/e3ff9329883b4ca9b853a7418d54c8c3, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:40,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/e3ff9329883b4ca9b853a7418d54c8c3 is 175, key is test_row_0/A:col10/1732765300121/Put/seqid=0 2024-11-28T03:41:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T03:41:40,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742007_1183 (size=39749) 2024-11-28T03:41:40,643 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/e3ff9329883b4ca9b853a7418d54c8c3 2024-11-28T03:41:40,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T03:41:40,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:40,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:40,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:40,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:40,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:40,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/09136770ae2945f7873bcd907e2136e3 is 50, key is test_row_0/B:col10/1732765300121/Put/seqid=0 2024-11-28T03:41:40,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742008_1184 (size=12151) 2024-11-28T03:41:40,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765360737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765360737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765360737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765360796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T03:41:40,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:40,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:40,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:40,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:40,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T03:41:40,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765360806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:40,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T03:41:40,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:40,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:40,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:40,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:40,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T03:41:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:41,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T03:41:41,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/09136770ae2945f7873bcd907e2136e3 2024-11-28T03:41:41,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/2a3acbc231a944989fa65625149af74b is 50, key is test_row_0/C:col10/1732765300121/Put/seqid=0 2024-11-28T03:41:41,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742009_1185 (size=12151) 2024-11-28T03:41:41,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765361256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:41,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765361256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765361256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:41,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T03:41:41,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:41,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765361306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:41,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765361327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:41,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T03:41:41,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:41,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/2a3acbc231a944989fa65625149af74b 2024-11-28T03:41:41,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T03:41:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/e3ff9329883b4ca9b853a7418d54c8c3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/e3ff9329883b4ca9b853a7418d54c8c3 2024-11-28T03:41:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:41,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/e3ff9329883b4ca9b853a7418d54c8c3, entries=200, sequenceid=221, filesize=38.8 K 2024-11-28T03:41:41,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/09136770ae2945f7873bcd907e2136e3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09136770ae2945f7873bcd907e2136e3 2024-11-28T03:41:41,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09136770ae2945f7873bcd907e2136e3, entries=150, sequenceid=221, filesize=11.9 K 2024-11-28T03:41:41,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/2a3acbc231a944989fa65625149af74b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/2a3acbc231a944989fa65625149af74b 2024-11-28T03:41:41,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/2a3acbc231a944989fa65625149af74b, entries=150, sequenceid=221, filesize=11.9 K 2024-11-28T03:41:41,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for eebe95cbdb6239965b947cdbd88704ed in 1489ms, sequenceid=221, compaction requested=true 2024-11-28T03:41:41,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:41,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:41,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:41,615 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:41,615 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:41,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:41,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:41,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:41,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:41,624 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133254 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:41,624 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/A is initiating minor compaction (all files) 2024-11-28T03:41:41,624 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/A in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,624 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/b4d3822a7e17428384dc1d1fe914dd97, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/1a619ed78a254c0fa1ebf9397bee02c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/fe3388b1b7ec42f084539291557101fd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/e3ff9329883b4ca9b853a7418d54c8c3] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=130.1 K 2024-11-28T03:41:41,624 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,624 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/b4d3822a7e17428384dc1d1fe914dd97, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/1a619ed78a254c0fa1ebf9397bee02c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/fe3388b1b7ec42f084539291557101fd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/e3ff9329883b4ca9b853a7418d54c8c3] 2024-11-28T03:41:41,625 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:41,625 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/B is initiating minor compaction (all files) 2024-11-28T03:41:41,625 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/B in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,625 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/d3d52da06fdf4dc29992eb6bdbf5f19a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/2054df253e684f74a836184be700c3df, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/17fa0eff0bc0410ab533b299f0709d68, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09136770ae2945f7873bcd907e2136e3] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=47.7 K 2024-11-28T03:41:41,625 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4d3822a7e17428384dc1d1fe914dd97, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732765295959 2024-11-28T03:41:41,626 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a619ed78a254c0fa1ebf9397bee02c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732765297146 2024-11-28T03:41:41,626 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d52da06fdf4dc29992eb6bdbf5f19a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732765295959 2024-11-28T03:41:41,627 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2054df253e684f74a836184be700c3df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732765297146 2024-11-28T03:41:41,627 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe3388b1b7ec42f084539291557101fd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732765297801 2024-11-28T03:41:41,628 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 17fa0eff0bc0410ab533b299f0709d68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732765297801 2024-11-28T03:41:41,628 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3ff9329883b4ca9b853a7418d54c8c3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732765300090 2024-11-28T03:41:41,628 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 09136770ae2945f7873bcd907e2136e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732765300090 2024-11-28T03:41:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T03:41:41,640 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:41,653 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:41,653 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/ccd86056e00449ec9aaca19f49df79ed is 50, key is test_row_0/B:col10/1732765300121/Put/seqid=0 2024-11-28T03:41:41,665 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128901af0edaafd4892b1d90ff78ec7e995_eebe95cbdb6239965b947cdbd88704ed store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:41,668 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128901af0edaafd4892b1d90ff78ec7e995_eebe95cbdb6239965b947cdbd88704ed, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:41,669 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128901af0edaafd4892b1d90ff78ec7e995_eebe95cbdb6239965b947cdbd88704ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:41,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742010_1186 (size=12527) 2024-11-28T03:41:41,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742011_1187 (size=4469) 2024-11-28T03:41:41,702 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#A#compaction#159 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:41,703 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/8a089f19819e4c40b2882dde8c8dface is 175, key is test_row_0/A:col10/1732765300121/Put/seqid=0 2024-11-28T03:41:41,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742012_1188 (size=31481) 2024-11-28T03:41:41,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T03:41:41,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:41,744 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-28T03:41:41,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:41,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:41,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:41,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:41,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:41,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b27d476a1f4346fe86bc40dee2d52b52_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765300157/Put/seqid=0 2024-11-28T03:41:41,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742013_1189 (size=12304) 2024-11-28T03:41:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:41,789 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b27d476a1f4346fe86bc40dee2d52b52_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b27d476a1f4346fe86bc40dee2d52b52_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:41,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/bf6229dd0e9548659562a3b5d8724fa0, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:41,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/bf6229dd0e9548659562a3b5d8724fa0 is 175, key is test_row_0/A:col10/1732765300157/Put/seqid=0 2024-11-28T03:41:41,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742014_1190 (size=31105) 2024-11-28T03:41:41,832 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/bf6229dd0e9548659562a3b5d8724fa0 2024-11-28T03:41:41,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/34093610e78b4182a60f000d5d8e1f93 is 50, key is test_row_0/B:col10/1732765300157/Put/seqid=0 2024-11-28T03:41:41,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742015_1191 (size=12151) 2024-11-28T03:41:42,095 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/ccd86056e00449ec9aaca19f49df79ed as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/ccd86056e00449ec9aaca19f49df79ed 2024-11-28T03:41:42,106 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/B of eebe95cbdb6239965b947cdbd88704ed into ccd86056e00449ec9aaca19f49df79ed(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:42,106 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:42,107 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/B, priority=12, startTime=1732765301615; duration=0sec 2024-11-28T03:41:42,107 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:42,107 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:B 2024-11-28T03:41:42,107 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:41:42,108 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:41:42,109 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/C is initiating minor compaction (all files) 2024-11-28T03:41:42,109 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/C in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:42,109 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/07b8561270f54e43a24085f3589bc099, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/5fdba96fec8640c79f9236e96a6db50a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/890ebb910f9d4a489346eba136ace9f6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/2a3acbc231a944989fa65625149af74b] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=47.7 K 2024-11-28T03:41:42,109 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 07b8561270f54e43a24085f3589bc099, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732765295959 2024-11-28T03:41:42,109 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fdba96fec8640c79f9236e96a6db50a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732765297146 2024-11-28T03:41:42,110 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 890ebb910f9d4a489346eba136ace9f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732765297801 2024-11-28T03:41:42,110 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3acbc231a944989fa65625149af74b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732765300090 2024-11-28T03:41:42,120 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#C#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:42,121 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/02b940771490478c80e39cc5d71c7f8d is 50, key is test_row_0/C:col10/1732765300121/Put/seqid=0 2024-11-28T03:41:42,152 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/8a089f19819e4c40b2882dde8c8dface as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/8a089f19819e4c40b2882dde8c8dface 2024-11-28T03:41:42,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742016_1192 (size=12527) 2024-11-28T03:41:42,161 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/A of eebe95cbdb6239965b947cdbd88704ed into 8a089f19819e4c40b2882dde8c8dface(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:42,161 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:42,161 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/A, priority=12, startTime=1732765301614; duration=0sec 2024-11-28T03:41:42,161 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:42,161 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:A 2024-11-28T03:41:42,270 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/34093610e78b4182a60f000d5d8e1f93 2024-11-28T03:41:42,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/90ed78f160b54e93a593434caf574f7f is 50, key is test_row_0/C:col10/1732765300157/Put/seqid=0 2024-11-28T03:41:42,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742017_1193 (size=12151) 2024-11-28T03:41:42,309 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/90ed78f160b54e93a593434caf574f7f 2024-11-28T03:41:42,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/bf6229dd0e9548659562a3b5d8724fa0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/bf6229dd0e9548659562a3b5d8724fa0 2024-11-28T03:41:42,320 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/bf6229dd0e9548659562a3b5d8724fa0, entries=150, sequenceid=230, filesize=30.4 K 2024-11-28T03:41:42,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/34093610e78b4182a60f000d5d8e1f93 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/34093610e78b4182a60f000d5d8e1f93 2024-11-28T03:41:42,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732765302303, max timestamp=1732765302326Num uniques 859; 2024-11-28T03:41:42,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-28T03:41:42,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765302303, max timestamp=1732765302326Num uniques 859; 2024-11-28T03:41:42,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:41:42,335 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/34093610e78b4182a60f000d5d8e1f93, entries=150, sequenceid=230, filesize=11.9 K 2024-11-28T03:41:42,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/90ed78f160b54e93a593434caf574f7f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/90ed78f160b54e93a593434caf574f7f 2024-11-28T03:41:42,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765302303, max timestamp=1732765302326Num uniques 859; 2024-11-28T03:41:42,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:41:42,344 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/90ed78f160b54e93a593434caf574f7f, entries=150, sequenceid=230, filesize=11.9 K 2024-11-28T03:41:42,345 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=58.14 KB/59540 for eebe95cbdb6239965b947cdbd88704ed in 601ms, sequenceid=230, compaction requested=false 2024-11-28T03:41:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-28T03:41:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-28T03:41:42,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-28T03:41:42,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8460 sec 2024-11-28T03:41:42,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=71.56 KB heapSize=191.23 KB 2024-11-28T03:41:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:42,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:42,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:42,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:42,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8550 sec 2024-11-28T03:41:42,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:42,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:42,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:41:42,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112803eb4d9fbf824aeab5ed9c488119d638_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765302352/Put/seqid=0 2024-11-28T03:41:42,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742018_1194 (size=12404) 2024-11-28T03:41:42,402 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:42,407 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112803eb4d9fbf824aeab5ed9c488119d638_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112803eb4d9fbf824aeab5ed9c488119d638_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:42,409 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/64fd2ab39cf94c54b9270efa64a35c86, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:42,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/64fd2ab39cf94c54b9270efa64a35c86 is 175, key is test_row_0/A:col10/1732765302352/Put/seqid=0 2024-11-28T03:41:42,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765362433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765362434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765362434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765362436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742019_1195 (size=31205) 2024-11-28T03:41:42,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765362446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,562 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/02b940771490478c80e39cc5d71c7f8d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/02b940771490478c80e39cc5d71c7f8d 2024-11-28T03:41:42,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765362556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765362556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765362557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765362557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,569 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/C of eebe95cbdb6239965b947cdbd88704ed into 02b940771490478c80e39cc5d71c7f8d(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:42,569 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:42,569 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/C, priority=12, startTime=1732765301615; duration=0sec 2024-11-28T03:41:42,569 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:42,569 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:C 2024-11-28T03:41:42,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765362566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T03:41:42,647 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:42,648 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-28T03:41:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T03:41:42,650 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:42,651 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:42,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T03:41:42,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765362776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765362777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765362777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765362777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765362787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:42,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:42,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:42,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:42,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:42,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:42,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:42,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:42,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:42,857 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/64fd2ab39cf94c54b9270efa64a35c86 2024-11-28T03:41:42,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/0f7f56b277eb4e49b1f344f29b6ee73c is 50, key is test_row_0/B:col10/1732765302352/Put/seqid=0 2024-11-28T03:41:42,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742020_1196 (size=12251) 2024-11-28T03:41:42,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/0f7f56b277eb4e49b1f344f29b6ee73c 2024-11-28T03:41:42,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4f8f59979c6e481a806efbf474e66a2e is 50, key is test_row_0/C:col10/1732765302352/Put/seqid=0 2024-11-28T03:41:42,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742021_1197 (size=12251) 2024-11-28T03:41:42,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4f8f59979c6e481a806efbf474e66a2e 2024-11-28T03:41:42,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/64fd2ab39cf94c54b9270efa64a35c86 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/64fd2ab39cf94c54b9270efa64a35c86 2024-11-28T03:41:42,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:42,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:42,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:42,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:42,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:42,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:42,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/64fd2ab39cf94c54b9270efa64a35c86, entries=150, sequenceid=259, filesize=30.5 K 2024-11-28T03:41:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T03:41:42,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/0f7f56b277eb4e49b1f344f29b6ee73c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0f7f56b277eb4e49b1f344f29b6ee73c 2024-11-28T03:41:42,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0f7f56b277eb4e49b1f344f29b6ee73c, entries=150, sequenceid=259, filesize=12.0 K 2024-11-28T03:41:42,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4f8f59979c6e481a806efbf474e66a2e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f8f59979c6e481a806efbf474e66a2e 2024-11-28T03:41:42,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f8f59979c6e481a806efbf474e66a2e, entries=150, sequenceid=259, filesize=12.0 K 2024-11-28T03:41:42,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~211.11 KB/216176, currentSize=107.34 KB/109920 for eebe95cbdb6239965b947cdbd88704ed in 639ms, sequenceid=259, compaction requested=true 2024-11-28T03:41:42,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:42,993 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:42,994 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:42,994 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/A is initiating minor compaction (all files) 2024-11-28T03:41:42,994 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/A in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:42,994 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/8a089f19819e4c40b2882dde8c8dface, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/bf6229dd0e9548659562a3b5d8724fa0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/64fd2ab39cf94c54b9270efa64a35c86] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=91.6 K 2024-11-28T03:41:42,994 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:42,994 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/8a089f19819e4c40b2882dde8c8dface, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/bf6229dd0e9548659562a3b5d8724fa0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/64fd2ab39cf94c54b9270efa64a35c86] 2024-11-28T03:41:42,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:42,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:42,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:42,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:42,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:42,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:42,995 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:42,995 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a089f19819e4c40b2882dde8c8dface, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732765300090 2024-11-28T03:41:42,995 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf6229dd0e9548659562a3b5d8724fa0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732765300138 2024-11-28T03:41:42,996 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64fd2ab39cf94c54b9270efa64a35c86, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732765302326 2024-11-28T03:41:42,996 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36929 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:42,996 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/B is initiating minor compaction (all files) 2024-11-28T03:41:42,996 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/B in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:42,997 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/ccd86056e00449ec9aaca19f49df79ed, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/34093610e78b4182a60f000d5d8e1f93, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0f7f56b277eb4e49b1f344f29b6ee73c] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=36.1 K 2024-11-28T03:41:43,004 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd86056e00449ec9aaca19f49df79ed, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732765300090 2024-11-28T03:41:43,005 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 34093610e78b4182a60f000d5d8e1f93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732765300138 2024-11-28T03:41:43,005 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7f56b277eb4e49b1f344f29b6ee73c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732765302326 2024-11-28T03:41:43,025 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:43,046 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:43,047 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/59dba214c12743849eff09628083bbee is 50, key is test_row_0/B:col10/1732765302352/Put/seqid=0 2024-11-28T03:41:43,049 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128f9d72338eb0a45ebafb9d6bd26ebcbf3_eebe95cbdb6239965b947cdbd88704ed store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:43,051 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128f9d72338eb0a45ebafb9d6bd26ebcbf3_eebe95cbdb6239965b947cdbd88704ed, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:43,051 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f9d72338eb0a45ebafb9d6bd26ebcbf3_eebe95cbdb6239965b947cdbd88704ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:43,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742022_1198 (size=12729) 2024-11-28T03:41:43,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T03:41:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:43,107 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/59dba214c12743849eff09628083bbee as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/59dba214c12743849eff09628083bbee 2024-11-28T03:41:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:43,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742023_1199 (size=4469) 2024-11-28T03:41:43,118 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/B of eebe95cbdb6239965b947cdbd88704ed into 59dba214c12743849eff09628083bbee(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:43,118 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:43,119 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/B, priority=13, startTime=1732765302995; duration=0sec 2024-11-28T03:41:43,119 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:43,119 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:B 2024-11-28T03:41:43,119 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:43,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:43,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:43,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,121 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36929 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:43,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,121 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/C is initiating minor compaction (all files) 2024-11-28T03:41:43,121 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/C in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,121 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/02b940771490478c80e39cc5d71c7f8d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/90ed78f160b54e93a593434caf574f7f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f8f59979c6e481a806efbf474e66a2e] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=36.1 K 2024-11-28T03:41:43,121 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 02b940771490478c80e39cc5d71c7f8d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732765300090 2024-11-28T03:41:43,122 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 90ed78f160b54e93a593434caf574f7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732765300138 2024-11-28T03:41:43,122 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f8f59979c6e481a806efbf474e66a2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732765302326 2024-11-28T03:41:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289a31236a61a640eebbcaeeffafc8bfec_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765303104/Put/seqid=0 2024-11-28T03:41:43,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765363136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765363138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765363140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,167 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#C#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,168 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/78066fb7bce248e1b5d15cf956780db5 is 50, key is test_row_0/C:col10/1732765302352/Put/seqid=0 2024-11-28T03:41:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765363160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765363160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742025_1201 (size=12729) 2024-11-28T03:41:43,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742024_1200 (size=14994) 2024-11-28T03:41:43,250 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:43,265 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/78066fb7bce248e1b5d15cf956780db5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/78066fb7bce248e1b5d15cf956780db5 2024-11-28T03:41:43,266 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289a31236a61a640eebbcaeeffafc8bfec_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289a31236a61a640eebbcaeeffafc8bfec_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:43,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/0ee80aca83c24a22a1e661baa85ec405, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:43,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/0ee80aca83c24a22a1e661baa85ec405 is 175, key is test_row_0/A:col10/1732765303104/Put/seqid=0 2024-11-28T03:41:43,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765363267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T03:41:43,278 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/C of eebe95cbdb6239965b947cdbd88704ed into 78066fb7bce248e1b5d15cf956780db5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:43,278 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:43,278 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/C, priority=13, startTime=1732765302995; duration=0sec 2024-11-28T03:41:43,278 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:43,278 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:C 2024-11-28T03:41:43,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:43,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:43,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765363276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765363276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765363277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765363277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742026_1202 (size=39949) 2024-11-28T03:41:43,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:43,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:43,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765363477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765363487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765363487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765363496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765363496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,513 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#A#compaction#168 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:43,514 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/4df71eda5d794866a043e502a7affdaf is 175, key is test_row_0/A:col10/1732765302352/Put/seqid=0 2024-11-28T03:41:43,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742027_1203 (size=31683) 2024-11-28T03:41:43,574 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/4df71eda5d794866a043e502a7affdaf as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/4df71eda5d794866a043e502a7affdaf 2024-11-28T03:41:43,584 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/A of eebe95cbdb6239965b947cdbd88704ed into 4df71eda5d794866a043e502a7affdaf(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:43,584 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:43,584 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/A, priority=13, startTime=1732765302993; duration=0sec 2024-11-28T03:41:43,584 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:43,584 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:A 2024-11-28T03:41:43,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:43,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:43,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,729 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/0ee80aca83c24a22a1e661baa85ec405 2024-11-28T03:41:43,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/1acfce774eb048caa789adeb4a975aec is 50, key is test_row_0/B:col10/1732765303104/Put/seqid=0 2024-11-28T03:41:43,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742028_1204 (size=12301) 2024-11-28T03:41:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T03:41:43,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765363796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765363796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765363796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765363807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765363807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:43,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:43,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:43,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:43,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/1acfce774eb048caa789adeb4a975aec 2024-11-28T03:41:44,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/80b1d10404ed42babeb103cec1ef9a19 is 50, key is test_row_0/C:col10/1732765303104/Put/seqid=0 2024-11-28T03:41:44,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:44,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742029_1205 (size=12301) 2024-11-28T03:41:44,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/80b1d10404ed42babeb103cec1ef9a19 2024-11-28T03:41:44,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/0ee80aca83c24a22a1e661baa85ec405 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0ee80aca83c24a22a1e661baa85ec405 2024-11-28T03:41:44,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0ee80aca83c24a22a1e661baa85ec405, entries=200, sequenceid=280, filesize=39.0 K 2024-11-28T03:41:44,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/1acfce774eb048caa789adeb4a975aec as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1acfce774eb048caa789adeb4a975aec 2024-11-28T03:41:44,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1acfce774eb048caa789adeb4a975aec, entries=150, sequenceid=280, filesize=12.0 K 2024-11-28T03:41:44,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/80b1d10404ed42babeb103cec1ef9a19 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/80b1d10404ed42babeb103cec1ef9a19 2024-11-28T03:41:44,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/80b1d10404ed42babeb103cec1ef9a19, entries=150, sequenceid=280, filesize=12.0 K 2024-11-28T03:41:44,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for eebe95cbdb6239965b947cdbd88704ed in 1173ms, sequenceid=280, compaction requested=false 2024-11-28T03:41:44,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:44,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:41:44,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:44,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:44,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:44,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:44,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:44,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:44,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bd767fb68fa249f5b3b69d7d05ce415c_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765304314/Put/seqid=0 2024-11-28T03:41:44,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765364344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765364348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765364351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765364352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765364353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:44,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:44,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742030_1206 (size=12454) 2024-11-28T03:41:44,375 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:44,382 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bd767fb68fa249f5b3b69d7d05ce415c_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bd767fb68fa249f5b3b69d7d05ce415c_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:44,383 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/98497dd9af42424c92765adfdb555a7b, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:44,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/98497dd9af42424c92765adfdb555a7b is 175, key is test_row_0/A:col10/1732765304314/Put/seqid=0 2024-11-28T03:41:44,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742031_1207 (size=31255) 2024-11-28T03:41:44,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765364457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765364466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765364466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765364466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765364466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:44,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:44,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765364666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765364677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765364677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765364686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765364686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,795 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/98497dd9af42424c92765adfdb555a7b 2024-11-28T03:41:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T03:41:44,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/42f9caead63240ffbc4a4fb632bf6202 is 50, key is test_row_0/B:col10/1732765304314/Put/seqid=0 2024-11-28T03:41:44,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742032_1208 (size=12301) 2024-11-28T03:41:44,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:44,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:44,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:44,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:44,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:44,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765364977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765364986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765364986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:44,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765364996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765364996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:45,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:45,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:45,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:45,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/42f9caead63240ffbc4a4fb632bf6202 2024-11-28T03:41:45,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/6f8a4da2757f44ac8fbf3dabaee3b0b3 is 50, key is test_row_0/C:col10/1732765304314/Put/seqid=0 2024-11-28T03:41:45,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742033_1209 (size=12301) 2024-11-28T03:41:45,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:45,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:45,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:45,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765365496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:45,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765365496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:45,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765365496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:45,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765365517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:45,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765365517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:45,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:45,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/6f8a4da2757f44ac8fbf3dabaee3b0b3 2024-11-28T03:41:45,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/98497dd9af42424c92765adfdb555a7b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/98497dd9af42424c92765adfdb555a7b 2024-11-28T03:41:45,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/98497dd9af42424c92765adfdb555a7b, entries=150, sequenceid=298, filesize=30.5 K 2024-11-28T03:41:45,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/42f9caead63240ffbc4a4fb632bf6202 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/42f9caead63240ffbc4a4fb632bf6202 2024-11-28T03:41:45,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/42f9caead63240ffbc4a4fb632bf6202, entries=150, sequenceid=298, filesize=12.0 K 2024-11-28T03:41:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/6f8a4da2757f44ac8fbf3dabaee3b0b3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6f8a4da2757f44ac8fbf3dabaee3b0b3 2024-11-28T03:41:45,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6f8a4da2757f44ac8fbf3dabaee3b0b3, entries=150, sequenceid=298, filesize=12.0 K 2024-11-28T03:41:45,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for eebe95cbdb6239965b947cdbd88704ed in 1385ms, sequenceid=298, compaction requested=true 2024-11-28T03:41:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:45,700 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:45,700 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:45,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:45,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:45,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:45,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:45,702 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102887 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:45,702 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/A is initiating minor compaction (all files) 2024-11-28T03:41:45,702 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/A in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,702 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/4df71eda5d794866a043e502a7affdaf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0ee80aca83c24a22a1e661baa85ec405, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/98497dd9af42424c92765adfdb555a7b] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=100.5 K 2024-11-28T03:41:45,702 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,702 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/4df71eda5d794866a043e502a7affdaf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0ee80aca83c24a22a1e661baa85ec405, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/98497dd9af42424c92765adfdb555a7b] 2024-11-28T03:41:45,703 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:45,703 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4df71eda5d794866a043e502a7affdaf, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732765302326 2024-11-28T03:41:45,703 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/B is initiating minor compaction (all files) 2024-11-28T03:41:45,703 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/B in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,703 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/59dba214c12743849eff09628083bbee, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1acfce774eb048caa789adeb4a975aec, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/42f9caead63240ffbc4a4fb632bf6202] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=36.5 K 2024-11-28T03:41:45,703 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ee80aca83c24a22a1e661baa85ec405, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732765302391 2024-11-28T03:41:45,703 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 59dba214c12743849eff09628083bbee, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732765302326 2024-11-28T03:41:45,704 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1acfce774eb048caa789adeb4a975aec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732765302391 2024-11-28T03:41:45,704 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98497dd9af42424c92765adfdb555a7b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732765303156 2024-11-28T03:41:45,704 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 42f9caead63240ffbc4a4fb632bf6202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732765303156 2024-11-28T03:41:45,727 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:45,729 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/98ce5ef86c8842df94642d41eeb5a4c6 is 50, key is test_row_0/B:col10/1732765304314/Put/seqid=0 2024-11-28T03:41:45,743 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:45,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T03:41:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,755 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:41:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:45,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742034_1210 (size=12881) 2024-11-28T03:41:45,785 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112865e9cbe408694d699b6a07f1764abbfb_eebe95cbdb6239965b947cdbd88704ed store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:45,788 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112865e9cbe408694d699b6a07f1764abbfb_eebe95cbdb6239965b947cdbd88704ed, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:45,788 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112865e9cbe408694d699b6a07f1764abbfb_eebe95cbdb6239965b947cdbd88704ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:45,795 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/98ce5ef86c8842df94642d41eeb5a4c6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/98ce5ef86c8842df94642d41eeb5a4c6 2024-11-28T03:41:45,802 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/B of eebe95cbdb6239965b947cdbd88704ed into 98ce5ef86c8842df94642d41eeb5a4c6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:45,802 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:45,803 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/B, priority=13, startTime=1732765305700; duration=0sec 2024-11-28T03:41:45,803 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:45,803 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:B 2024-11-28T03:41:45,803 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:45,804 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:45,804 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/C is initiating minor compaction (all files) 2024-11-28T03:41:45,804 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/C in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:45,804 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/78066fb7bce248e1b5d15cf956780db5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/80b1d10404ed42babeb103cec1ef9a19, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6f8a4da2757f44ac8fbf3dabaee3b0b3] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=36.5 K 2024-11-28T03:41:45,806 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 78066fb7bce248e1b5d15cf956780db5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732765302326 2024-11-28T03:41:45,806 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 80b1d10404ed42babeb103cec1ef9a19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732765302391 2024-11-28T03:41:45,807 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f8a4da2757f44ac8fbf3dabaee3b0b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732765303156 2024-11-28T03:41:45,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285d6b9e6196eb40bc8b5917f514e8abae_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765304350/Put/seqid=0 2024-11-28T03:41:45,822 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#C#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:45,822 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/fb439cfbbe05462897dc1d398d3f664b is 50, key is test_row_0/C:col10/1732765304314/Put/seqid=0 2024-11-28T03:41:45,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742035_1211 (size=4469) 2024-11-28T03:41:45,847 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#A#compaction#178 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:45,848 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/2ad8ea837d024163bd6337fd33d89367 is 175, key is test_row_0/A:col10/1732765304314/Put/seqid=0 2024-11-28T03:41:45,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742036_1212 (size=12454) 2024-11-28T03:41:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:45,879 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285d6b9e6196eb40bc8b5917f514e8abae_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285d6b9e6196eb40bc8b5917f514e8abae_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:45,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742038_1214 (size=31835) 2024-11-28T03:41:45,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/f8599fbc347c4efb9e728ef2760dcabd, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:45,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/f8599fbc347c4efb9e728ef2760dcabd is 175, key is test_row_0/A:col10/1732765304350/Put/seqid=0 2024-11-28T03:41:45,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742037_1213 (size=12881) 2024-11-28T03:41:45,895 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/2ad8ea837d024163bd6337fd33d89367 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/2ad8ea837d024163bd6337fd33d89367 2024-11-28T03:41:45,895 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/fb439cfbbe05462897dc1d398d3f664b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fb439cfbbe05462897dc1d398d3f664b 2024-11-28T03:41:45,913 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/A of eebe95cbdb6239965b947cdbd88704ed into 2ad8ea837d024163bd6337fd33d89367(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:45,913 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:45,913 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/A, priority=13, startTime=1732765305700; duration=0sec 2024-11-28T03:41:45,913 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:45,913 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:A 2024-11-28T03:41:45,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742039_1215 (size=31255) 2024-11-28T03:41:45,916 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/C of eebe95cbdb6239965b947cdbd88704ed into fb439cfbbe05462897dc1d398d3f664b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:45,916 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:45,916 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/C, priority=13, startTime=1732765305701; duration=0sec 2024-11-28T03:41:45,916 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:45,916 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:C 2024-11-28T03:41:46,315 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/f8599fbc347c4efb9e728ef2760dcabd 2024-11-28T03:41:46,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/09b10fc0186c4b87a38f1ee3e69c451f is 50, key is test_row_0/B:col10/1732765304350/Put/seqid=0 2024-11-28T03:41:46,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742040_1216 (size=12301) 2024-11-28T03:41:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:46,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:46,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765366556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765366554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765366557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765366559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765366560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765366666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765366666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765366676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765366676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765366677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,784 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/09b10fc0186c4b87a38f1ee3e69c451f 2024-11-28T03:41:46,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/29d498defc87461493c346bb3cb016ad is 50, key is test_row_0/C:col10/1732765304350/Put/seqid=0 2024-11-28T03:41:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T03:41:46,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742041_1217 (size=12301) 2024-11-28T03:41:46,830 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/29d498defc87461493c346bb3cb016ad 2024-11-28T03:41:46,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/f8599fbc347c4efb9e728ef2760dcabd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/f8599fbc347c4efb9e728ef2760dcabd 2024-11-28T03:41:46,847 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/f8599fbc347c4efb9e728ef2760dcabd, entries=150, sequenceid=316, filesize=30.5 K 2024-11-28T03:41:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/09b10fc0186c4b87a38f1ee3e69c451f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09b10fc0186c4b87a38f1ee3e69c451f 2024-11-28T03:41:46,861 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09b10fc0186c4b87a38f1ee3e69c451f, entries=150, sequenceid=316, filesize=12.0 K 2024-11-28T03:41:46,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/29d498defc87461493c346bb3cb016ad as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/29d498defc87461493c346bb3cb016ad 2024-11-28T03:41:46,869 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/29d498defc87461493c346bb3cb016ad, entries=150, sequenceid=316, filesize=12.0 K 2024-11-28T03:41:46,870 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for eebe95cbdb6239965b947cdbd88704ed in 1116ms, sequenceid=316, compaction requested=false 2024-11-28T03:41:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-28T03:41:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-28T03:41:46,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-28T03:41:46,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.2220 sec 2024-11-28T03:41:46,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 4.2300 sec 2024-11-28T03:41:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:46,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T03:41:46,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:46,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:46,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:46,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:46,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:46,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:46,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ce0433450059469a8dcb46a4b229f4b1_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765306880/Put/seqid=0 2024-11-28T03:41:46,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742042_1218 (size=14994) 2024-11-28T03:41:46,906 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:46,915 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ce0433450059469a8dcb46a4b229f4b1_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ce0433450059469a8dcb46a4b229f4b1_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:46,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765366908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,916 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765366910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef is 175, key is test_row_0/A:col10/1732765306880/Put/seqid=0 2024-11-28T03:41:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765366914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765366914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765366916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:46,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742043_1219 (size=39949) 2024-11-28T03:41:47,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765367026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765367026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765367026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765367027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765367027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765367237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765367237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765367237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765367237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765367237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,353 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef 2024-11-28T03:41:47,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/1533e5993cd147fea7285e19269a9823 is 50, key is test_row_0/B:col10/1732765306880/Put/seqid=0 2024-11-28T03:41:47,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742044_1220 (size=12301) 2024-11-28T03:41:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765367556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765367557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765367557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765367557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765367557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:47,604 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T03:41:47,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/1533e5993cd147fea7285e19269a9823 2024-11-28T03:41:47,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/0452b3604b9b4e68a5177f43fd71b09c is 50, key is test_row_0/C:col10/1732765306880/Put/seqid=0 2024-11-28T03:41:47,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742045_1221 (size=12301) 2024-11-28T03:41:48,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765368066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:48,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765368067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:48,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765368067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:48,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765368067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:48,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765368077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:48,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/0452b3604b9b4e68a5177f43fd71b09c 2024-11-28T03:41:48,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef 2024-11-28T03:41:48,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef, entries=200, sequenceid=335, filesize=39.0 K 2024-11-28T03:41:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/1533e5993cd147fea7285e19269a9823 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1533e5993cd147fea7285e19269a9823 2024-11-28T03:41:48,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1533e5993cd147fea7285e19269a9823, entries=150, sequenceid=335, filesize=12.0 K 2024-11-28T03:41:48,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/0452b3604b9b4e68a5177f43fd71b09c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/0452b3604b9b4e68a5177f43fd71b09c 2024-11-28T03:41:48,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/0452b3604b9b4e68a5177f43fd71b09c, entries=150, sequenceid=335, filesize=12.0 K 2024-11-28T03:41:48,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for eebe95cbdb6239965b947cdbd88704ed in 1393ms, sequenceid=335, compaction requested=true 2024-11-28T03:41:48,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:48,277 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:48,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:48,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:48,279 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:48,279 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:48,279 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/A is initiating minor compaction (all files) 2024-11-28T03:41:48,279 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/A in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:48,279 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/2ad8ea837d024163bd6337fd33d89367, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/f8599fbc347c4efb9e728ef2760dcabd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=100.6 K 2024-11-28T03:41:48,279 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:48,279 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/2ad8ea837d024163bd6337fd33d89367, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/f8599fbc347c4efb9e728ef2760dcabd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef] 2024-11-28T03:41:48,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:48,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:48,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:48,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:48,280 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad8ea837d024163bd6337fd33d89367, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732765303156 2024-11-28T03:41:48,280 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:48,280 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8599fbc347c4efb9e728ef2760dcabd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732765304343 2024-11-28T03:41:48,280 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/B is initiating minor compaction (all files) 2024-11-28T03:41:48,280 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/B in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:48,280 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/98ce5ef86c8842df94642d41eeb5a4c6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09b10fc0186c4b87a38f1ee3e69c451f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1533e5993cd147fea7285e19269a9823] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=36.6 K 2024-11-28T03:41:48,281 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ce5ef86c8842df94642d41eeb5a4c6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732765303156 2024-11-28T03:41:48,281 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba4fae4f2c994ec1b9dafe7a8f1a1fef, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732765306524 2024-11-28T03:41:48,284 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 09b10fc0186c4b87a38f1ee3e69c451f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732765304343 2024-11-28T03:41:48,285 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1533e5993cd147fea7285e19269a9823, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732765306524 2024-11-28T03:41:48,293 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:48,298 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#B#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:48,299 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/8b18106f30ea4bd88f82827497396e99 is 50, key is test_row_0/B:col10/1732765306880/Put/seqid=0 2024-11-28T03:41:48,304 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411282a0d85b5564a41c58bd1bf93d91d25f9_eebe95cbdb6239965b947cdbd88704ed store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:48,307 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411282a0d85b5564a41c58bd1bf93d91d25f9_eebe95cbdb6239965b947cdbd88704ed, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:48,307 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282a0d85b5564a41c58bd1bf93d91d25f9_eebe95cbdb6239965b947cdbd88704ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:48,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742046_1222 (size=12983) 2024-11-28T03:41:48,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742047_1223 (size=4469) 2024-11-28T03:41:48,724 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/8b18106f30ea4bd88f82827497396e99 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8b18106f30ea4bd88f82827497396e99 2024-11-28T03:41:48,730 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/B of eebe95cbdb6239965b947cdbd88704ed into 8b18106f30ea4bd88f82827497396e99(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:48,731 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:48,731 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/B, priority=13, startTime=1732765308279; duration=0sec 2024-11-28T03:41:48,731 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:48,731 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:B 2024-11-28T03:41:48,731 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:48,732 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:48,732 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/C is initiating minor compaction (all files) 2024-11-28T03:41:48,732 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/C in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:48,732 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fb439cfbbe05462897dc1d398d3f664b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/29d498defc87461493c346bb3cb016ad, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/0452b3604b9b4e68a5177f43fd71b09c] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=36.6 K 2024-11-28T03:41:48,733 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting fb439cfbbe05462897dc1d398d3f664b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732765303156 2024-11-28T03:41:48,734 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 29d498defc87461493c346bb3cb016ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732765304343 2024-11-28T03:41:48,735 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0452b3604b9b4e68a5177f43fd71b09c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732765306524 2024-11-28T03:41:48,743 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#A#compaction#186 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:48,745 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/780d5777a2bb4cb8a778a2ba27059d96 is 175, key is test_row_0/A:col10/1732765306880/Put/seqid=0 2024-11-28T03:41:48,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742048_1224 (size=31937) 2024-11-28T03:41:48,762 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#C#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:48,763 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/d04b84cea2b649839d61599318002b89 is 50, key is test_row_0/C:col10/1732765306880/Put/seqid=0 2024-11-28T03:41:48,768 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/780d5777a2bb4cb8a778a2ba27059d96 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/780d5777a2bb4cb8a778a2ba27059d96 2024-11-28T03:41:48,779 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/A of eebe95cbdb6239965b947cdbd88704ed into 780d5777a2bb4cb8a778a2ba27059d96(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:48,779 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:48,779 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/A, priority=13, startTime=1732765308277; duration=0sec 2024-11-28T03:41:48,780 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:48,780 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:A 2024-11-28T03:41:48,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742049_1225 (size=12983) 2024-11-28T03:41:48,828 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/d04b84cea2b649839d61599318002b89 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/d04b84cea2b649839d61599318002b89 2024-11-28T03:41:48,835 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/C of eebe95cbdb6239965b947cdbd88704ed into d04b84cea2b649839d61599318002b89(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:48,835 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:48,835 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/C, priority=13, startTime=1732765308279; duration=0sec 2024-11-28T03:41:48,835 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:48,835 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:C 2024-11-28T03:41:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:49,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T03:41:49,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:49,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:49,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:49,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:49,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:49,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:49,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112837cba612d62445b1adc7f869697974d7_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765306907/Put/seqid=0 2024-11-28T03:41:49,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742050_1226 (size=14994) 2024-11-28T03:41:49,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765369102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765369102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765369106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765369114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765369121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765369217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765369217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765369226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765369227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765369227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765369426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765369426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765369436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765369437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765369437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,510 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:49,517 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112837cba612d62445b1adc7f869697974d7_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112837cba612d62445b1adc7f869697974d7_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:49,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/682b519e47f14d3e938ae5c4c7ae44c9, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:49,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/682b519e47f14d3e938ae5c4c7ae44c9 is 175, key is test_row_0/A:col10/1732765306907/Put/seqid=0 2024-11-28T03:41:49,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742051_1227 (size=39949) 2024-11-28T03:41:49,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765369736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765369736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765369747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765369747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765369747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:49,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/682b519e47f14d3e938ae5c4c7ae44c9 2024-11-28T03:41:49,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/719f7e00b93f4053a87738cbe4e21c0b is 50, key is test_row_0/B:col10/1732765306907/Put/seqid=0 2024-11-28T03:41:49,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742052_1228 (size=12301) 2024-11-28T03:41:49,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/719f7e00b93f4053a87738cbe4e21c0b 2024-11-28T03:41:50,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f2afe82572c64fbe8abb5fa87bd631e7 is 50, key is test_row_0/C:col10/1732765306907/Put/seqid=0 2024-11-28T03:41:50,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742053_1229 (size=12301) 2024-11-28T03:41:50,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f2afe82572c64fbe8abb5fa87bd631e7 2024-11-28T03:41:50,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/682b519e47f14d3e938ae5c4c7ae44c9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/682b519e47f14d3e938ae5c4c7ae44c9 2024-11-28T03:41:50,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/682b519e47f14d3e938ae5c4c7ae44c9, entries=200, sequenceid=355, filesize=39.0 K 2024-11-28T03:41:50,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/719f7e00b93f4053a87738cbe4e21c0b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/719f7e00b93f4053a87738cbe4e21c0b 2024-11-28T03:41:50,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/719f7e00b93f4053a87738cbe4e21c0b, entries=150, sequenceid=355, filesize=12.0 K 2024-11-28T03:41:50,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f2afe82572c64fbe8abb5fa87bd631e7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f2afe82572c64fbe8abb5fa87bd631e7 2024-11-28T03:41:50,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f2afe82572c64fbe8abb5fa87bd631e7, entries=150, sequenceid=355, filesize=12.0 K 2024-11-28T03:41:50,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for eebe95cbdb6239965b947cdbd88704ed in 969ms, sequenceid=355, compaction requested=false 2024-11-28T03:41:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:50,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:50,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T03:41:50,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:50,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:50,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:50,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:50,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:50,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:50,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287735a6aee5664483937356e18f916384_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765309101/Put/seqid=0 2024-11-28T03:41:50,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765370274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765370276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765370277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742054_1230 (size=12454) 2024-11-28T03:41:50,288 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:50,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765370280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765370284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287735a6aee5664483937356e18f916384_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287735a6aee5664483937356e18f916384_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,294 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/84b15d13a7dc4a81bfe528fa1ea3ed02, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:50,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/84b15d13a7dc4a81bfe528fa1ea3ed02 is 175, key is test_row_0/A:col10/1732765309101/Put/seqid=0 2024-11-28T03:41:50,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742055_1231 (size=31255) 2024-11-28T03:41:50,305 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/84b15d13a7dc4a81bfe528fa1ea3ed02 2024-11-28T03:41:50,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/0892046325ec4b9e837041537c14df7b is 50, key is test_row_0/B:col10/1732765309101/Put/seqid=0 2024-11-28T03:41:50,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742056_1232 (size=12301) 2024-11-28T03:41:50,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/0892046325ec4b9e837041537c14df7b 2024-11-28T03:41:50,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4dd6098a56304842aecdf47b98aff951 is 50, key is test_row_0/C:col10/1732765309101/Put/seqid=0 2024-11-28T03:41:50,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742057_1233 (size=12301) 2024-11-28T03:41:50,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765370386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765370386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765370397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765370397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765370397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765370596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765370596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765370607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765370607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765370607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4dd6098a56304842aecdf47b98aff951 2024-11-28T03:41:50,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/84b15d13a7dc4a81bfe528fa1ea3ed02 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/84b15d13a7dc4a81bfe528fa1ea3ed02 2024-11-28T03:41:50,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/84b15d13a7dc4a81bfe528fa1ea3ed02, entries=150, sequenceid=372, filesize=30.5 K 2024-11-28T03:41:50,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/0892046325ec4b9e837041537c14df7b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0892046325ec4b9e837041537c14df7b 2024-11-28T03:41:50,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0892046325ec4b9e837041537c14df7b, entries=150, sequenceid=372, filesize=12.0 K 2024-11-28T03:41:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4dd6098a56304842aecdf47b98aff951 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4dd6098a56304842aecdf47b98aff951 2024-11-28T03:41:50,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4dd6098a56304842aecdf47b98aff951, entries=150, sequenceid=372, filesize=12.0 K 2024-11-28T03:41:50,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for eebe95cbdb6239965b947cdbd88704ed in 530ms, sequenceid=372, compaction requested=true 2024-11-28T03:41:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:41:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:50,779 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:50,779 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:41:50,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:50,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eebe95cbdb6239965b947cdbd88704ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:41:50,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:50,780 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:50,781 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/A is initiating minor compaction (all files) 2024-11-28T03:41:50,781 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/A in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:50,781 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/780d5777a2bb4cb8a778a2ba27059d96, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/682b519e47f14d3e938ae5c4c7ae44c9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/84b15d13a7dc4a81bfe528fa1ea3ed02] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=100.7 K 2024-11-28T03:41:50,781 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:50,781 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/780d5777a2bb4cb8a778a2ba27059d96, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/682b519e47f14d3e938ae5c4c7ae44c9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/84b15d13a7dc4a81bfe528fa1ea3ed02] 2024-11-28T03:41:50,789 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:50,789 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/B is initiating minor compaction (all files) 2024-11-28T03:41:50,789 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/B in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:50,789 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8b18106f30ea4bd88f82827497396e99, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/719f7e00b93f4053a87738cbe4e21c0b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0892046325ec4b9e837041537c14df7b] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=36.7 K 2024-11-28T03:41:50,789 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 780d5777a2bb4cb8a778a2ba27059d96, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732765306524 2024-11-28T03:41:50,789 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b18106f30ea4bd88f82827497396e99, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732765306524 2024-11-28T03:41:50,790 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 719f7e00b93f4053a87738cbe4e21c0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732765306907 2024-11-28T03:41:50,790 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 682b519e47f14d3e938ae5c4c7ae44c9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732765306907 2024-11-28T03:41:50,790 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0892046325ec4b9e837041537c14df7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732765309101 2024-11-28T03:41:50,791 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84b15d13a7dc4a81bfe528fa1ea3ed02, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732765309101 2024-11-28T03:41:50,810 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:50,811 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/26c538e7d5a14659bdd4303b573497a6 is 50, key is test_row_0/B:col10/1732765309101/Put/seqid=0 2024-11-28T03:41:50,817 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:50,821 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112800f9a64b464646a48ae11bef1b3c0876_eebe95cbdb6239965b947cdbd88704ed store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:50,823 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112800f9a64b464646a48ae11bef1b3c0876_eebe95cbdb6239965b947cdbd88704ed, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:50,823 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112800f9a64b464646a48ae11bef1b3c0876_eebe95cbdb6239965b947cdbd88704ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:50,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742058_1234 (size=13085) 2024-11-28T03:41:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T03:41:50,858 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:50,859 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-28T03:41:50,864 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T03:41:50,865 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:50,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:50,867 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/26c538e7d5a14659bdd4303b573497a6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/26c538e7d5a14659bdd4303b573497a6 2024-11-28T03:41:50,875 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/B of eebe95cbdb6239965b947cdbd88704ed into 26c538e7d5a14659bdd4303b573497a6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:50,875 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:50,875 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/B, priority=13, startTime=1732765310779; duration=0sec 2024-11-28T03:41:50,875 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:41:50,875 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:B 2024-11-28T03:41:50,876 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:41:50,877 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:41:50,877 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eebe95cbdb6239965b947cdbd88704ed/C is initiating minor compaction (all files) 2024-11-28T03:41:50,877 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eebe95cbdb6239965b947cdbd88704ed/C in TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:50,877 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/d04b84cea2b649839d61599318002b89, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f2afe82572c64fbe8abb5fa87bd631e7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4dd6098a56304842aecdf47b98aff951] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp, totalSize=36.7 K 2024-11-28T03:41:50,879 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d04b84cea2b649839d61599318002b89, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732765306524 2024-11-28T03:41:50,880 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting f2afe82572c64fbe8abb5fa87bd631e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732765306907 2024-11-28T03:41:50,881 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd6098a56304842aecdf47b98aff951, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732765309101 2024-11-28T03:41:50,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742059_1235 (size=4469) 2024-11-28T03:41:50,918 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#C#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:50,919 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/7a7681b658cb4a99866a6076413a0fcb is 50, key is test_row_0/C:col10/1732765309101/Put/seqid=0 2024-11-28T03:41:50,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T03:41:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:50,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:50,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:50,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:50,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:50,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:50,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:50,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765370957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765370959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765370961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765370966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:50,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742060_1236 (size=13085) 2024-11-28T03:41:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T03:41:50,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288897c494194f4a81aea809c01f0ec54b_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765310278/Put/seqid=0 2024-11-28T03:41:50,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765370966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742061_1237 (size=17534) 2024-11-28T03:41:51,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T03:41:51,020 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:51,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:51,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,025 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288897c494194f4a81aea809c01f0ec54b_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288897c494194f4a81aea809c01f0ec54b_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:51,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/9fbf25f9bf7d4211ad3ce079a5b8b45b, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:51,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/9fbf25f9bf7d4211ad3ce079a5b8b45b is 175, key is test_row_0/A:col10/1732765310278/Put/seqid=0 2024-11-28T03:41:51,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742062_1238 (size=48639) 2024-11-28T03:41:51,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765371077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765371077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765371077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765371077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765371096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T03:41:51,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:51,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T03:41:51,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765371286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765371297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765371297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,307 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eebe95cbdb6239965b947cdbd88704ed#A#compaction#196 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765371297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,308 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/d0e303e7349d4193be8964608c173c4d is 175, key is test_row_0/A:col10/1732765309101/Put/seqid=0 2024-11-28T03:41:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765371320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T03:41:51,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:51,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742063_1239 (size=32039) 2024-11-28T03:41:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,339 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/d0e303e7349d4193be8964608c173c4d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/d0e303e7349d4193be8964608c173c4d 2024-11-28T03:41:51,346 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/A of eebe95cbdb6239965b947cdbd88704ed into d0e303e7349d4193be8964608c173c4d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:51,346 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:51,346 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/A, priority=13, startTime=1732765310779; duration=0sec 2024-11-28T03:41:51,346 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:51,346 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:A 2024-11-28T03:41:51,389 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/7a7681b658cb4a99866a6076413a0fcb as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/7a7681b658cb4a99866a6076413a0fcb 2024-11-28T03:41:51,398 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eebe95cbdb6239965b947cdbd88704ed/C of eebe95cbdb6239965b947cdbd88704ed into 7a7681b658cb4a99866a6076413a0fcb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:41:51,398 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:51,398 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., storeName=eebe95cbdb6239965b947cdbd88704ed/C, priority=13, startTime=1732765310780; duration=0sec 2024-11-28T03:41:51,398 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:41:51,398 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eebe95cbdb6239965b947cdbd88704ed:C 2024-11-28T03:41:51,466 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/9fbf25f9bf7d4211ad3ce079a5b8b45b 2024-11-28T03:41:51,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/ca08b4cb7a2e4c359d4dadc52c77addf is 50, key is test_row_0/B:col10/1732765310278/Put/seqid=0 2024-11-28T03:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T03:41:51,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T03:41:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742064_1240 (size=12301) 2024-11-28T03:41:51,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765371606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765371617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765371617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765371617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765371626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:51,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T03:41:51,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:51,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,786 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-11-28T03:41:51,786 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:51,786 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:51,786 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:51,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:51,787 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-11-28T03:41:51,787 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:51,787 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:51,787 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:51,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:51,788 INFO [Thread-715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-715. 2024-11-28T03:41:51,789 DEBUG [Thread-715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:51,789 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:51,789 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:51,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:51,796 INFO [Thread-713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-713. 2024-11-28T03:41:51,796 DEBUG [Thread-713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:51,796 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:51,796 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:51,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:51,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T03:41:51,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:51,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/ca08b4cb7a2e4c359d4dadc52c77addf 2024-11-28T03:41:51,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4e1ef050c838451dbfd735499fcbd166 is 50, key is test_row_0/C:col10/1732765310278/Put/seqid=0 2024-11-28T03:41:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742065_1241 (size=12301) 2024-11-28T03:41:51,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T03:41:51,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:51,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T03:41:52,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T03:41:52,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:52,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:52,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:52,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:52,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:52,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46070 deadline: 1732765372116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:52,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:52,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46048 deadline: 1732765372126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46068 deadline: 1732765372126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:52,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46090 deadline: 1732765372127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:52,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46054 deadline: 1732765372147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:41:52,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T03:41:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:52,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:52,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:52,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4e1ef050c838451dbfd735499fcbd166 2024-11-28T03:41:52,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/9fbf25f9bf7d4211ad3ce079a5b8b45b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/9fbf25f9bf7d4211ad3ce079a5b8b45b 2024-11-28T03:41:52,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/9fbf25f9bf7d4211ad3ce079a5b8b45b, entries=250, sequenceid=391, filesize=47.5 K 2024-11-28T03:41:52,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/ca08b4cb7a2e4c359d4dadc52c77addf as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/ca08b4cb7a2e4c359d4dadc52c77addf 2024-11-28T03:41:52,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/ca08b4cb7a2e4c359d4dadc52c77addf, entries=150, sequenceid=391, filesize=12.0 K 2024-11-28T03:41:52,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/4e1ef050c838451dbfd735499fcbd166 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4e1ef050c838451dbfd735499fcbd166 2024-11-28T03:41:52,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4e1ef050c838451dbfd735499fcbd166, entries=150, sequenceid=391, filesize=12.0 K 2024-11-28T03:41:52,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for eebe95cbdb6239965b947cdbd88704ed in 1416ms, sequenceid=391, compaction requested=false 2024-11-28T03:41:52,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:52,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T03:41:52,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:52,423 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T03:41:52,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:52,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:52,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:52,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:52,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:52,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:52,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d496b4ef7c3542f2b365df4f72abd580_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765310955/Put/seqid=0 2024-11-28T03:41:52,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742066_1242 (size=12454) 2024-11-28T03:41:52,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:52,838 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d496b4ef7c3542f2b365df4f72abd580_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d496b4ef7c3542f2b365df4f72abd580_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:52,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/5305994e740f45e0bb1dca5d8163abb0, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:52,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/5305994e740f45e0bb1dca5d8163abb0 is 175, key is test_row_0/A:col10/1732765310955/Put/seqid=0 2024-11-28T03:41:52,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742067_1243 (size=31255) 2024-11-28T03:41:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T03:41:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:53,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. as already flushing 2024-11-28T03:41:53,128 INFO [Thread-700 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-700. 2024-11-28T03:41:53,128 DEBUG [Thread-700 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:53,128 DEBUG [Thread-700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:53,128 DEBUG [Thread-700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:53,128 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:53,138 INFO [Thread-702 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-702. 2024-11-28T03:41:53,138 INFO [Thread-706 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-706. 2024-11-28T03:41:53,138 DEBUG [Thread-706 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:53,138 DEBUG [Thread-702 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:53,138 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:53,138 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:53,138 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:53,138 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:53,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:53,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:53,147 INFO [Thread-704 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-704. 2024-11-28T03:41:53,147 DEBUG [Thread-704 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:53,147 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:53,147 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:53,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:53,157 INFO [Thread-698 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-698. 2024-11-28T03:41:53,157 DEBUG [Thread-698 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:41:53,157 DEBUG [Thread-698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:53,157 DEBUG [Thread-698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:53,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:53,244 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/5305994e740f45e0bb1dca5d8163abb0 2024-11-28T03:41:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/6e73c1dff3404d42bd03e399d558eebb is 50, key is test_row_0/B:col10/1732765310955/Put/seqid=0 2024-11-28T03:41:53,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742068_1244 (size=12301) 2024-11-28T03:41:53,656 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/6e73c1dff3404d42bd03e399d558eebb 2024-11-28T03:41:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f2210c91982347c1ac4cb36f33ec8531 is 50, key is test_row_0/C:col10/1732765310955/Put/seqid=0 2024-11-28T03:41:53,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742069_1245 (size=12301) 2024-11-28T03:41:54,068 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f2210c91982347c1ac4cb36f33ec8531 2024-11-28T03:41:54,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/5305994e740f45e0bb1dca5d8163abb0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/5305994e740f45e0bb1dca5d8163abb0 2024-11-28T03:41:54,077 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/5305994e740f45e0bb1dca5d8163abb0, entries=150, sequenceid=408, filesize=30.5 K 2024-11-28T03:41:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/6e73c1dff3404d42bd03e399d558eebb as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/6e73c1dff3404d42bd03e399d558eebb 2024-11-28T03:41:54,082 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/6e73c1dff3404d42bd03e399d558eebb, entries=150, sequenceid=408, filesize=12.0 K 2024-11-28T03:41:54,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f2210c91982347c1ac4cb36f33ec8531 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f2210c91982347c1ac4cb36f33ec8531 2024-11-28T03:41:54,086 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f2210c91982347c1ac4cb36f33ec8531, entries=150, sequenceid=408, filesize=12.0 K 2024-11-28T03:41:54,087 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=33.54 KB/34350 for eebe95cbdb6239965b947cdbd88704ed in 1664ms, sequenceid=408, compaction requested=true 2024-11-28T03:41:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for eebe95cbdb6239965b947cdbd88704ed: 2024-11-28T03:41:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-28T03:41:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-28T03:41:54,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-28T03:41:54,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2230 sec 2024-11-28T03:41:54,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 3.2310 sec 2024-11-28T03:41:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T03:41:55,027 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2472 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2511 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1121 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3363 rows 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1132 2024-11-28T03:41:55,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3389 rows 2024-11-28T03:41:55,027 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:41:55,027 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:41:55,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:55,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:55,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:41:55,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T03:41:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T03:41:55,031 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765315031"}]},"ts":"1732765315031"} 2024-11-28T03:41:55,033 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T03:41:55,033 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T03:41:55,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T03:41:55,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, UNASSIGN}] 2024-11-28T03:41:55,035 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, UNASSIGN 2024-11-28T03:41:55,036 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=eebe95cbdb6239965b947cdbd88704ed, regionState=CLOSING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:41:55,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, UNASSIGN because future has completed 2024-11-28T03:41:55,037 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T03:41:55,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:41:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T03:41:55,189 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:55,189 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T03:41:55,190 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing eebe95cbdb6239965b947cdbd88704ed, disabling compactions & flushes 2024-11-28T03:41:55,190 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:55,190 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:55,190 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. after waiting 0 ms 2024-11-28T03:41:55,190 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:55,190 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing eebe95cbdb6239965b947cdbd88704ed 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T03:41:55,190 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=A 2024-11-28T03:41:55,190 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:55,190 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=B 2024-11-28T03:41:55,190 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:55,190 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eebe95cbdb6239965b947cdbd88704ed, store=C 2024-11-28T03:41:55,190 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:55,196 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e9385c21c68d49938aa93920e66da429_eebe95cbdb6239965b947cdbd88704ed is 50, key is test_row_0/A:col10/1732765313137/Put/seqid=0 2024-11-28T03:41:55,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742070_1246 (size=12454) 2024-11-28T03:41:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T03:41:55,601 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:55,605 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e9385c21c68d49938aa93920e66da429_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e9385c21c68d49938aa93920e66da429_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:55,606 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/4e761672333f4689b78394dcdb9a59f2, store: [table=TestAcidGuarantees family=A region=eebe95cbdb6239965b947cdbd88704ed] 2024-11-28T03:41:55,607 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/4e761672333f4689b78394dcdb9a59f2 is 175, key is test_row_0/A:col10/1732765313137/Put/seqid=0 2024-11-28T03:41:55,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742071_1247 (size=31255) 2024-11-28T03:41:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T03:41:56,012 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/4e761672333f4689b78394dcdb9a59f2 2024-11-28T03:41:56,020 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/04c0039cf0924173b8ea94e35154e1e4 is 50, key is test_row_0/B:col10/1732765313137/Put/seqid=0 2024-11-28T03:41:56,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742072_1248 (size=12301) 2024-11-28T03:41:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T03:41:56,424 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/04c0039cf0924173b8ea94e35154e1e4 2024-11-28T03:41:56,431 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f859b2238863446682be3c3060dd9afd is 50, key is test_row_0/C:col10/1732765313137/Put/seqid=0 2024-11-28T03:41:56,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742073_1249 (size=12301) 2024-11-28T03:41:56,835 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f859b2238863446682be3c3060dd9afd 2024-11-28T03:41:56,840 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/A/4e761672333f4689b78394dcdb9a59f2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/4e761672333f4689b78394dcdb9a59f2 2024-11-28T03:41:56,843 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/4e761672333f4689b78394dcdb9a59f2, entries=150, sequenceid=416, filesize=30.5 K 2024-11-28T03:41:56,844 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/B/04c0039cf0924173b8ea94e35154e1e4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/04c0039cf0924173b8ea94e35154e1e4 2024-11-28T03:41:56,847 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/04c0039cf0924173b8ea94e35154e1e4, entries=150, sequenceid=416, filesize=12.0 K 2024-11-28T03:41:56,848 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/.tmp/C/f859b2238863446682be3c3060dd9afd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f859b2238863446682be3c3060dd9afd 2024-11-28T03:41:56,851 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f859b2238863446682be3c3060dd9afd, entries=150, sequenceid=416, filesize=12.0 K 2024-11-28T03:41:56,852 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for eebe95cbdb6239965b947cdbd88704ed in 1662ms, sequenceid=416, compaction requested=true 2024-11-28T03:41:56,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/d9de935f02d04ecdbe919e46105b41cb, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/71171ad6e007428895385b66fc6f38f2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0aea09312f9d4c158e38322d58280f0b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/cca8c4ab9447434ea901b1fa8c3e353d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c8f1b579f1674894b7d16efe794d6862, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c302a6a09c9442188d1b024ae3d7b8c0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/b4d3822a7e17428384dc1d1fe914dd97, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/43f8a65f64f844199c43201f5454faf2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/1a619ed78a254c0fa1ebf9397bee02c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/fe3388b1b7ec42f084539291557101fd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/e3ff9329883b4ca9b853a7418d54c8c3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/8a089f19819e4c40b2882dde8c8dface, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/bf6229dd0e9548659562a3b5d8724fa0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/4df71eda5d794866a043e502a7affdaf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/64fd2ab39cf94c54b9270efa64a35c86, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0ee80aca83c24a22a1e661baa85ec405, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/2ad8ea837d024163bd6337fd33d89367, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/98497dd9af42424c92765adfdb555a7b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/f8599fbc347c4efb9e728ef2760dcabd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/780d5777a2bb4cb8a778a2ba27059d96, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/682b519e47f14d3e938ae5c4c7ae44c9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/84b15d13a7dc4a81bfe528fa1ea3ed02] to archive 2024-11-28T03:41:56,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:41:56,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/d9de935f02d04ecdbe919e46105b41cb to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/d9de935f02d04ecdbe919e46105b41cb 2024-11-28T03:41:56,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/71171ad6e007428895385b66fc6f38f2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/71171ad6e007428895385b66fc6f38f2 2024-11-28T03:41:56,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0aea09312f9d4c158e38322d58280f0b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0aea09312f9d4c158e38322d58280f0b 2024-11-28T03:41:56,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/cca8c4ab9447434ea901b1fa8c3e353d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/cca8c4ab9447434ea901b1fa8c3e353d 2024-11-28T03:41:56,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c8f1b579f1674894b7d16efe794d6862 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c8f1b579f1674894b7d16efe794d6862 2024-11-28T03:41:56,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c302a6a09c9442188d1b024ae3d7b8c0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/c302a6a09c9442188d1b024ae3d7b8c0 2024-11-28T03:41:56,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/b4d3822a7e17428384dc1d1fe914dd97 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/b4d3822a7e17428384dc1d1fe914dd97 2024-11-28T03:41:56,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/43f8a65f64f844199c43201f5454faf2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/43f8a65f64f844199c43201f5454faf2 2024-11-28T03:41:56,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/1a619ed78a254c0fa1ebf9397bee02c4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/1a619ed78a254c0fa1ebf9397bee02c4 2024-11-28T03:41:56,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/fe3388b1b7ec42f084539291557101fd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/fe3388b1b7ec42f084539291557101fd 2024-11-28T03:41:56,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/e3ff9329883b4ca9b853a7418d54c8c3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/e3ff9329883b4ca9b853a7418d54c8c3 2024-11-28T03:41:56,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/8a089f19819e4c40b2882dde8c8dface to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/8a089f19819e4c40b2882dde8c8dface 2024-11-28T03:41:56,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/bf6229dd0e9548659562a3b5d8724fa0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/bf6229dd0e9548659562a3b5d8724fa0 2024-11-28T03:41:56,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/4df71eda5d794866a043e502a7affdaf to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/4df71eda5d794866a043e502a7affdaf 2024-11-28T03:41:56,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/64fd2ab39cf94c54b9270efa64a35c86 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/64fd2ab39cf94c54b9270efa64a35c86 2024-11-28T03:41:56,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0ee80aca83c24a22a1e661baa85ec405 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/0ee80aca83c24a22a1e661baa85ec405 2024-11-28T03:41:56,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/2ad8ea837d024163bd6337fd33d89367 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/2ad8ea837d024163bd6337fd33d89367 2024-11-28T03:41:56,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/98497dd9af42424c92765adfdb555a7b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/98497dd9af42424c92765adfdb555a7b 2024-11-28T03:41:56,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/f8599fbc347c4efb9e728ef2760dcabd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/f8599fbc347c4efb9e728ef2760dcabd 2024-11-28T03:41:56,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/ba4fae4f2c994ec1b9dafe7a8f1a1fef 2024-11-28T03:41:56,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/780d5777a2bb4cb8a778a2ba27059d96 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/780d5777a2bb4cb8a778a2ba27059d96 2024-11-28T03:41:56,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/682b519e47f14d3e938ae5c4c7ae44c9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/682b519e47f14d3e938ae5c4c7ae44c9 2024-11-28T03:41:56,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/84b15d13a7dc4a81bfe528fa1ea3ed02 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/84b15d13a7dc4a81bfe528fa1ea3ed02 2024-11-28T03:41:56,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/b24f188b840949ee9e34ead8c6a8b4b0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e3240e83a13e446ebb5163dc731a1c3e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c409f570180c400c81fd77ad27854da7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/7d76df2e22274dd8a121e34fcf313b8c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c784bcaeaa424afb96dc505d4b1f9b39, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8cf4dd48ea594e4f85ad3de28e1e7427, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/d3d52da06fdf4dc29992eb6bdbf5f19a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e92a205406a24b96bac8dda6ffcc8feb, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/2054df253e684f74a836184be700c3df, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/17fa0eff0bc0410ab533b299f0709d68, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/ccd86056e00449ec9aaca19f49df79ed, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09136770ae2945f7873bcd907e2136e3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/34093610e78b4182a60f000d5d8e1f93, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/59dba214c12743849eff09628083bbee, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0f7f56b277eb4e49b1f344f29b6ee73c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1acfce774eb048caa789adeb4a975aec, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/98ce5ef86c8842df94642d41eeb5a4c6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/42f9caead63240ffbc4a4fb632bf6202, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09b10fc0186c4b87a38f1ee3e69c451f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8b18106f30ea4bd88f82827497396e99, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1533e5993cd147fea7285e19269a9823, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/719f7e00b93f4053a87738cbe4e21c0b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0892046325ec4b9e837041537c14df7b] to archive 2024-11-28T03:41:56,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:41:56,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/b24f188b840949ee9e34ead8c6a8b4b0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/b24f188b840949ee9e34ead8c6a8b4b0 2024-11-28T03:41:56,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e3240e83a13e446ebb5163dc731a1c3e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e3240e83a13e446ebb5163dc731a1c3e 2024-11-28T03:41:56,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c409f570180c400c81fd77ad27854da7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c409f570180c400c81fd77ad27854da7 2024-11-28T03:41:56,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/7d76df2e22274dd8a121e34fcf313b8c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/7d76df2e22274dd8a121e34fcf313b8c 2024-11-28T03:41:56,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c784bcaeaa424afb96dc505d4b1f9b39 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/c784bcaeaa424afb96dc505d4b1f9b39 2024-11-28T03:41:56,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8cf4dd48ea594e4f85ad3de28e1e7427 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8cf4dd48ea594e4f85ad3de28e1e7427 2024-11-28T03:41:56,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/d3d52da06fdf4dc29992eb6bdbf5f19a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/d3d52da06fdf4dc29992eb6bdbf5f19a 2024-11-28T03:41:56,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e92a205406a24b96bac8dda6ffcc8feb to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/e92a205406a24b96bac8dda6ffcc8feb 2024-11-28T03:41:56,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/2054df253e684f74a836184be700c3df to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/2054df253e684f74a836184be700c3df 2024-11-28T03:41:56,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/17fa0eff0bc0410ab533b299f0709d68 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/17fa0eff0bc0410ab533b299f0709d68 2024-11-28T03:41:56,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/ccd86056e00449ec9aaca19f49df79ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/ccd86056e00449ec9aaca19f49df79ed 2024-11-28T03:41:56,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09136770ae2945f7873bcd907e2136e3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09136770ae2945f7873bcd907e2136e3 2024-11-28T03:41:56,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/34093610e78b4182a60f000d5d8e1f93 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/34093610e78b4182a60f000d5d8e1f93 2024-11-28T03:41:56,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/59dba214c12743849eff09628083bbee to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/59dba214c12743849eff09628083bbee 2024-11-28T03:41:56,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0f7f56b277eb4e49b1f344f29b6ee73c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0f7f56b277eb4e49b1f344f29b6ee73c 2024-11-28T03:41:56,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1acfce774eb048caa789adeb4a975aec to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1acfce774eb048caa789adeb4a975aec 2024-11-28T03:41:56,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/98ce5ef86c8842df94642d41eeb5a4c6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/98ce5ef86c8842df94642d41eeb5a4c6 2024-11-28T03:41:56,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/42f9caead63240ffbc4a4fb632bf6202 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/42f9caead63240ffbc4a4fb632bf6202 2024-11-28T03:41:56,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09b10fc0186c4b87a38f1ee3e69c451f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/09b10fc0186c4b87a38f1ee3e69c451f 2024-11-28T03:41:56,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8b18106f30ea4bd88f82827497396e99 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/8b18106f30ea4bd88f82827497396e99 2024-11-28T03:41:56,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1533e5993cd147fea7285e19269a9823 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/1533e5993cd147fea7285e19269a9823 2024-11-28T03:41:56,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/719f7e00b93f4053a87738cbe4e21c0b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/719f7e00b93f4053a87738cbe4e21c0b 2024-11-28T03:41:56,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0892046325ec4b9e837041537c14df7b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/0892046325ec4b9e837041537c14df7b 2024-11-28T03:41:56,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6b1aebbabf6e47cfad03c875f3be9cb0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/50c5401f619f42eda77316d31339b192, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fade6c51120c48d39a34b5be7b73b6d9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f7a4b8b5276c46c4b3b24988bec7efa8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/dd20e2bf311243efa1a6b8f3106d7d10, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f667817fd444ea4801e33f9813a9de9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/07b8561270f54e43a24085f3589bc099, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/eaa25eec4e244225bb3bb7fa1f4c4bec, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/5fdba96fec8640c79f9236e96a6db50a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/890ebb910f9d4a489346eba136ace9f6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/02b940771490478c80e39cc5d71c7f8d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/2a3acbc231a944989fa65625149af74b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/90ed78f160b54e93a593434caf574f7f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/78066fb7bce248e1b5d15cf956780db5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f8f59979c6e481a806efbf474e66a2e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/80b1d10404ed42babeb103cec1ef9a19, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fb439cfbbe05462897dc1d398d3f664b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6f8a4da2757f44ac8fbf3dabaee3b0b3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/29d498defc87461493c346bb3cb016ad, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/d04b84cea2b649839d61599318002b89, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/0452b3604b9b4e68a5177f43fd71b09c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f2afe82572c64fbe8abb5fa87bd631e7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4dd6098a56304842aecdf47b98aff951] to archive 2024-11-28T03:41:56,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:41:56,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6b1aebbabf6e47cfad03c875f3be9cb0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6b1aebbabf6e47cfad03c875f3be9cb0 2024-11-28T03:41:56,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/50c5401f619f42eda77316d31339b192 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/50c5401f619f42eda77316d31339b192 2024-11-28T03:41:56,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fade6c51120c48d39a34b5be7b73b6d9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fade6c51120c48d39a34b5be7b73b6d9 2024-11-28T03:41:56,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f7a4b8b5276c46c4b3b24988bec7efa8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f7a4b8b5276c46c4b3b24988bec7efa8 2024-11-28T03:41:56,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/dd20e2bf311243efa1a6b8f3106d7d10 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/dd20e2bf311243efa1a6b8f3106d7d10 2024-11-28T03:41:56,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f667817fd444ea4801e33f9813a9de9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f667817fd444ea4801e33f9813a9de9 2024-11-28T03:41:56,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/07b8561270f54e43a24085f3589bc099 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/07b8561270f54e43a24085f3589bc099 2024-11-28T03:41:56,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/eaa25eec4e244225bb3bb7fa1f4c4bec to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/eaa25eec4e244225bb3bb7fa1f4c4bec 2024-11-28T03:41:56,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/5fdba96fec8640c79f9236e96a6db50a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/5fdba96fec8640c79f9236e96a6db50a 2024-11-28T03:41:56,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/890ebb910f9d4a489346eba136ace9f6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/890ebb910f9d4a489346eba136ace9f6 2024-11-28T03:41:56,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/02b940771490478c80e39cc5d71c7f8d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/02b940771490478c80e39cc5d71c7f8d 2024-11-28T03:41:56,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/2a3acbc231a944989fa65625149af74b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/2a3acbc231a944989fa65625149af74b 2024-11-28T03:41:56,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/90ed78f160b54e93a593434caf574f7f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/90ed78f160b54e93a593434caf574f7f 2024-11-28T03:41:56,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/78066fb7bce248e1b5d15cf956780db5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/78066fb7bce248e1b5d15cf956780db5 2024-11-28T03:41:56,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f8f59979c6e481a806efbf474e66a2e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4f8f59979c6e481a806efbf474e66a2e 2024-11-28T03:41:56,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/80b1d10404ed42babeb103cec1ef9a19 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/80b1d10404ed42babeb103cec1ef9a19 2024-11-28T03:41:56,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fb439cfbbe05462897dc1d398d3f664b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/fb439cfbbe05462897dc1d398d3f664b 2024-11-28T03:41:56,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6f8a4da2757f44ac8fbf3dabaee3b0b3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/6f8a4da2757f44ac8fbf3dabaee3b0b3 2024-11-28T03:41:56,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/29d498defc87461493c346bb3cb016ad to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/29d498defc87461493c346bb3cb016ad 2024-11-28T03:41:56,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/d04b84cea2b649839d61599318002b89 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/d04b84cea2b649839d61599318002b89 2024-11-28T03:41:56,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/0452b3604b9b4e68a5177f43fd71b09c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/0452b3604b9b4e68a5177f43fd71b09c 2024-11-28T03:41:56,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f2afe82572c64fbe8abb5fa87bd631e7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f2afe82572c64fbe8abb5fa87bd631e7 2024-11-28T03:41:56,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4dd6098a56304842aecdf47b98aff951 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4dd6098a56304842aecdf47b98aff951 2024-11-28T03:41:56,963 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/recovered.edits/419.seqid, newMaxSeqId=419, maxSeqId=4 2024-11-28T03:41:56,964 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. 2024-11-28T03:41:56,964 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for eebe95cbdb6239965b947cdbd88704ed: Waiting for close lock at 1732765315190Running coprocessor pre-close hooks at 1732765315190Disabling compacts and flushes for region at 1732765315190Disabling writes for close at 1732765315190Obtaining lock to block concurrent updates at 1732765315190Preparing flush snapshotting stores in eebe95cbdb6239965b947cdbd88704ed at 1732765315190Finished memstore snapshotting TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732765315190Flushing stores of TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed. at 1732765315191 (+1 ms)Flushing eebe95cbdb6239965b947cdbd88704ed/A: creating writer at 1732765315191Flushing mob file eebe95cbdb6239965b947cdbd88704ed/A: appending metadata at 1732765315196 (+5 ms)Flushing mob file eebe95cbdb6239965b947cdbd88704ed/A: closing flushed file at 1732765315196Flushing eebe95cbdb6239965b947cdbd88704ed/A: appending metadata at 1732765315606 (+410 ms)Flushing eebe95cbdb6239965b947cdbd88704ed/A: closing flushed file at 1732765315606Flushing eebe95cbdb6239965b947cdbd88704ed/B: creating writer at 1732765316016 (+410 ms)Flushing eebe95cbdb6239965b947cdbd88704ed/B: appending metadata at 1732765316019 (+3 ms)Flushing eebe95cbdb6239965b947cdbd88704ed/B: closing flushed file at 1732765316019Flushing eebe95cbdb6239965b947cdbd88704ed/C: creating writer at 1732765316428 (+409 ms)Flushing eebe95cbdb6239965b947cdbd88704ed/C: appending metadata at 1732765316430 (+2 ms)Flushing eebe95cbdb6239965b947cdbd88704ed/C: closing flushed file at 1732765316430Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@aecd843: reopening flushed file at 1732765316839 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dcdb326: reopening flushed file at 1732765316843 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bdcd0fa: reopening flushed file at 1732765316847 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for eebe95cbdb6239965b947cdbd88704ed in 1662ms, sequenceid=416, compaction requested=true at 1732765316852 (+5 ms)Writing region close event to WAL at 1732765316959 (+107 ms)Running coprocessor post-close hooks at 1732765316963 (+4 ms)Closed at 1732765316964 (+1 ms) 2024-11-28T03:41:56,966 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:56,966 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=eebe95cbdb6239965b947cdbd88704ed, regionState=CLOSED 2024-11-28T03:41:56,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:41:56,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-28T03:41:56,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure eebe95cbdb6239965b947cdbd88704ed, server=3a546af21488,34671,1732765249447 in 1.9320 sec 2024-11-28T03:41:56,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-28T03:41:56,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eebe95cbdb6239965b947cdbd88704ed, UNASSIGN in 1.9360 sec 2024-11-28T03:41:56,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-28T03:41:56,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9400 sec 2024-11-28T03:41:56,976 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765316976"}]},"ts":"1732765316976"} 2024-11-28T03:41:56,978 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T03:41:56,978 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T03:41:56,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9500 sec 2024-11-28T03:41:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T03:41:57,177 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:57,178 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T03:41:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:57,179 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T03:41:57,180 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:57,182 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,184 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/recovered.edits] 2024-11-28T03:41:57,187 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/4e761672333f4689b78394dcdb9a59f2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/4e761672333f4689b78394dcdb9a59f2 2024-11-28T03:41:57,188 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/5305994e740f45e0bb1dca5d8163abb0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/5305994e740f45e0bb1dca5d8163abb0 2024-11-28T03:41:57,189 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/9fbf25f9bf7d4211ad3ce079a5b8b45b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/9fbf25f9bf7d4211ad3ce079a5b8b45b 2024-11-28T03:41:57,190 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/d0e303e7349d4193be8964608c173c4d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/A/d0e303e7349d4193be8964608c173c4d 2024-11-28T03:41:57,192 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/04c0039cf0924173b8ea94e35154e1e4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/04c0039cf0924173b8ea94e35154e1e4 2024-11-28T03:41:57,193 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/26c538e7d5a14659bdd4303b573497a6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/26c538e7d5a14659bdd4303b573497a6 2024-11-28T03:41:57,194 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/6e73c1dff3404d42bd03e399d558eebb to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/6e73c1dff3404d42bd03e399d558eebb 2024-11-28T03:41:57,195 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/ca08b4cb7a2e4c359d4dadc52c77addf to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/B/ca08b4cb7a2e4c359d4dadc52c77addf 2024-11-28T03:41:57,197 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4e1ef050c838451dbfd735499fcbd166 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/4e1ef050c838451dbfd735499fcbd166 2024-11-28T03:41:57,198 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/7a7681b658cb4a99866a6076413a0fcb to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/7a7681b658cb4a99866a6076413a0fcb 2024-11-28T03:41:57,199 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f2210c91982347c1ac4cb36f33ec8531 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f2210c91982347c1ac4cb36f33ec8531 2024-11-28T03:41:57,200 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f859b2238863446682be3c3060dd9afd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/C/f859b2238863446682be3c3060dd9afd 2024-11-28T03:41:57,202 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/recovered.edits/419.seqid to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed/recovered.edits/419.seqid 2024-11-28T03:41:57,203 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,203 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T03:41:57,203 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T03:41:57,204 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T03:41:57,207 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112803d284421ae7479ab7a6f489da52ca97_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112803d284421ae7479ab7a6f489da52ca97_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,208 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112803eb4d9fbf824aeab5ed9c488119d638_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112803eb4d9fbf824aeab5ed9c488119d638_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,209 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128106441e3c43a4ed38d739ec192cc566a_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128106441e3c43a4ed38d739ec192cc566a_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,210 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112837cba612d62445b1adc7f869697974d7_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112837cba612d62445b1adc7f869697974d7_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,211 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128490908877d594d488e4ebcbd7f3be149_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128490908877d594d488e4ebcbd7f3be149_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,212 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285d6b9e6196eb40bc8b5917f514e8abae_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285d6b9e6196eb40bc8b5917f514e8abae_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,213 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287735a6aee5664483937356e18f916384_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287735a6aee5664483937356e18f916384_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,215 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288897c494194f4a81aea809c01f0ec54b_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288897c494194f4a81aea809c01f0ec54b_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,217 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289a31236a61a640eebbcaeeffafc8bfec_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289a31236a61a640eebbcaeeffafc8bfec_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,218 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128af88d97ec0fe4ed2b1ac3cf5c0b8a995_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128af88d97ec0fe4ed2b1ac3cf5c0b8a995_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,219 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b27d476a1f4346fe86bc40dee2d52b52_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b27d476a1f4346fe86bc40dee2d52b52_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,220 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bd767fb68fa249f5b3b69d7d05ce415c_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bd767fb68fa249f5b3b69d7d05ce415c_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,221 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c1162b9acedb462e9d98423bd340a1c7_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c1162b9acedb462e9d98423bd340a1c7_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,222 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ce0433450059469a8dcb46a4b229f4b1_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ce0433450059469a8dcb46a4b229f4b1_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,224 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d496b4ef7c3542f2b365df4f72abd580_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d496b4ef7c3542f2b365df4f72abd580_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,225 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128de54851322c646798ec8dae6b47be647_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128de54851322c646798ec8dae6b47be647_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,226 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e9385c21c68d49938aa93920e66da429_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e9385c21c68d49938aa93920e66da429_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,227 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ee43c593418b4df291510bd65a07c6b4_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ee43c593418b4df291510bd65a07c6b4_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,229 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fdbbba6108064509889ab49ff5d3d924_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fdbbba6108064509889ab49ff5d3d924_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,230 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fe157ab495e8483ab865d3ea3d551136_eebe95cbdb6239965b947cdbd88704ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fe157ab495e8483ab865d3ea3d551136_eebe95cbdb6239965b947cdbd88704ed 2024-11-28T03:41:57,231 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T03:41:57,233 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:57,235 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T03:41:57,238 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T03:41:57,239 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:57,239 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T03:41:57,239 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732765317239"}]},"ts":"9223372036854775807"} 2024-11-28T03:41:57,241 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T03:41:57,242 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => eebe95cbdb6239965b947cdbd88704ed, NAME => 'TestAcidGuarantees,,1732765289437.eebe95cbdb6239965b947cdbd88704ed.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T03:41:57,242 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T03:41:57,242 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732765317242"}]},"ts":"9223372036854775807"} 2024-11-28T03:41:57,243 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T03:41:57,244 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:57,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-28T03:41:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T03:41:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T03:41:57,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:57,296 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 246) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/cluster_bc0417c7-3664-f9fe-1add-573adc96de9e/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_764232134_22 at /127.0.0.1:39132 [Waiting for operation #107] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-769855082_22 at /127.0.0.1:39722 [Waiting for operation #207] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_764232134_22 at /127.0.0.1:39630 [Waiting for operation #237] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/cluster_bc0417c7-3664-f9fe-1add-573adc96de9e/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=526 (was 528), ProcessCount=11 (was 11), AvailableMemoryMB=6880 (was 7507) 2024-11-28T03:41:57,305 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=526, ProcessCount=11, AvailableMemoryMB=6880 2024-11-28T03:41:57,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T03:41:57,306 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:41:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T03:41:57,308 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:41:57,309 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:57,309 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-28T03:41:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T03:41:57,309 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:41:57,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742074_1250 (size=1061) 2024-11-28T03:41:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T03:41:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T03:41:57,717 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72 2024-11-28T03:41:57,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742075_1251 (size=53) 2024-11-28T03:41:57,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T03:41:58,123 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:41:58,123 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1a2c0de76d7e5c66916b7f6387695a75, disabling compactions & flushes 2024-11-28T03:41:58,123 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:58,123 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:58,123 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. after waiting 0 ms 2024-11-28T03:41:58,123 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:58,123 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:58,123 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a2c0de76d7e5c66916b7f6387695a75: Waiting for close lock at 1732765318123Disabling compacts and flushes for region at 1732765318123Disabling writes for close at 1732765318123Writing region close event to WAL at 1732765318123Closed at 1732765318123 2024-11-28T03:41:58,125 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:41:58,125 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732765318125"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732765318125"}]},"ts":"1732765318125"} 2024-11-28T03:41:58,126 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:41:58,127 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:41:58,127 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765318127"}]},"ts":"1732765318127"} 2024-11-28T03:41:58,128 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T03:41:58,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a2c0de76d7e5c66916b7f6387695a75, ASSIGN}] 2024-11-28T03:41:58,130 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a2c0de76d7e5c66916b7f6387695a75, ASSIGN 2024-11-28T03:41:58,130 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a2c0de76d7e5c66916b7f6387695a75, ASSIGN; state=OFFLINE, location=3a546af21488,34671,1732765249447; forceNewPlan=false, retain=false 2024-11-28T03:41:58,281 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=1a2c0de76d7e5c66916b7f6387695a75, regionState=OPENING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:41:58,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a2c0de76d7e5c66916b7f6387695a75, ASSIGN because future has completed 2024-11-28T03:41:58,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:41:58,438 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:58,438 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:41:58,438 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,438 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:41:58,438 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,438 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,439 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,441 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:41:58,441 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a2c0de76d7e5c66916b7f6387695a75 columnFamilyName A 2024-11-28T03:41:58,441 DEBUG [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:58,442 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] regionserver.HStore(327): Store=1a2c0de76d7e5c66916b7f6387695a75/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:41:58,442 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,443 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:41:58,443 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a2c0de76d7e5c66916b7f6387695a75 columnFamilyName B 2024-11-28T03:41:58,443 DEBUG [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:58,443 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] regionserver.HStore(327): Store=1a2c0de76d7e5c66916b7f6387695a75/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:41:58,443 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,444 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:41:58,444 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a2c0de76d7e5c66916b7f6387695a75 columnFamilyName C 2024-11-28T03:41:58,444 DEBUG [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:41:58,445 INFO [StoreOpener-1a2c0de76d7e5c66916b7f6387695a75-1 {}] regionserver.HStore(327): Store=1a2c0de76d7e5c66916b7f6387695a75/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:41:58,445 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:58,445 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,445 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,446 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T03:41:58,447 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,447 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,447 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:41:58,448 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,450 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:41:58,451 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 1a2c0de76d7e5c66916b7f6387695a75; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59346336, jitterRate=-0.11567068099975586}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:41:58,451 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:58,451 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 1a2c0de76d7e5c66916b7f6387695a75: Running coprocessor pre-open hook at 1732765318438Writing region info on filesystem at 1732765318438Initializing all the Stores at 1732765318439 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765318439Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765318439Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765318439Cleaning up temporary data from old regions at 1732765318447 (+8 ms)Running coprocessor post-open hooks at 1732765318451 (+4 ms)Region opened successfully at 1732765318451 2024-11-28T03:41:58,452 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., pid=63, masterSystemTime=1732765318434 2024-11-28T03:41:58,454 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:58,454 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:58,454 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=1a2c0de76d7e5c66916b7f6387695a75, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:41:58,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:41:58,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-28T03:41:58,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 in 173 msec 2024-11-28T03:41:58,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-28T03:41:58,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a2c0de76d7e5c66916b7f6387695a75, ASSIGN in 328 msec 2024-11-28T03:41:58,459 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:41:58,459 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765318459"}]},"ts":"1732765318459"} 2024-11-28T03:41:58,461 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T03:41:58,461 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:41:58,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-28T03:41:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T03:41:59,457 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:41:59,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e45e9d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,458 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,459 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a841234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,462 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b80567, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,463 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,465 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T03:41:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 3a546af21488,38327,1732765248628 2024-11-28T03:41:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4239ccfa 2024-11-28T03:41:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T03:41:59,467 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T03:41:59,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd53738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,468 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f22e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,470 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43444dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31898a93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,471 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d3938b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,472 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a66d2d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f95838d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,474 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@726dd5f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,475 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c9a417e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a612003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,477 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aa3eba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,478 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b2dc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461a8f29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,479 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf6d7db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,480 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb35087, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e034cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,482 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5c31aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5afbe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a99c3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,484 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45af4d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,485 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce36a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28795458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,487 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc2b08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,488 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45afef27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@532ccbbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,490 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7062d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,491 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@785352e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4335b2cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:41:59,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:41:59,493 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:41:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:41:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:41:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e8e6d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:41:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:41:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:41:59,494 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:41:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7236fa6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:41:59,495 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,495 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,495 DEBUG [Thread-1174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,495 DEBUG [Thread-1183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,496 DEBUG [Thread-1183 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,496 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:41:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,496 DEBUG [Thread-1183 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,496 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-28T03:41:59,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T03:41:59,499 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:41:59,499 DEBUG [Thread-1181 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,500 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,500 DEBUG [Thread-1185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,500 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:41:59,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:41:59,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:41:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:41:59,504 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:41:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:41:59,504 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:41:59,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,505 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:41:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:41:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:41:59,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:41:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:41:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:41:59,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:41:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:41:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:41:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:41:59,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:41:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:41:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:41:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:41:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:41:59,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/ede0dfa327c04956a6499ab590513518 is 50, key is test_row_0/A:col10/1732765319506/Put/seqid=0 2024-11-28T03:41:59,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742076_1252 (size=12001) 2024-11-28T03:41:59,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765379545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732765319538, max timestamp=1732765319552Num uniques 859; 2024-11-28T03:41:59,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732765319538, max timestamp=1732765319552Num uniques 859; 2024-11-28T03:41:59,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:41:59,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:41:59,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732765319538, max timestamp=1732765319552Num uniques 859; 2024-11-28T03:41:59,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:41:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T03:41:59,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765379625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765379629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765379631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765379632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T03:41:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:41:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:59,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:59,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765379706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765379746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765379746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765379746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765379746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T03:41:59,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T03:41:59,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:41:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:59,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:59,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765379916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/ede0dfa327c04956a6499ab590513518 2024-11-28T03:41:59,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765379956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765379956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765379957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:41:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765379957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:41:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:41:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:41:59,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T03:41:59,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:59,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:41:59,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:41:59,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:59,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:41:59,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/c11430cbddec4f7b8b1a0fbe92cadf80 is 50, key is test_row_0/B:col10/1732765319506/Put/seqid=0 2024-11-28T03:41:59,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742077_1253 (size=12001) 2024-11-28T03:42:00,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T03:42:00,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:00,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T03:42:00,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765380226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765380267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765380267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765380267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:00,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765380267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:00,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T03:42:00,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/c11430cbddec4f7b8b1a0fbe92cadf80 2024-11-28T03:42:00,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/ab304b2a3e2940b7b174abf791e615b8 is 50, key is test_row_0/C:col10/1732765319506/Put/seqid=0 2024-11-28T03:42:00,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742078_1254 (size=12001) 2024-11-28T03:42:00,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T03:42:00,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:00,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T03:42:00,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:00,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T03:42:00,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T03:42:00,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:00,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765380736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:00,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765380776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765380776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:00,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765380786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765380787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:00,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/ab304b2a3e2940b7b174abf791e615b8 2024-11-28T03:42:00,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/ede0dfa327c04956a6499ab590513518 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/ede0dfa327c04956a6499ab590513518 2024-11-28T03:42:00,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/ede0dfa327c04956a6499ab590513518, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T03:42:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/c11430cbddec4f7b8b1a0fbe92cadf80 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c11430cbddec4f7b8b1a0fbe92cadf80 2024-11-28T03:42:00,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c11430cbddec4f7b8b1a0fbe92cadf80, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T03:42:00,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/ab304b2a3e2940b7b174abf791e615b8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ab304b2a3e2940b7b174abf791e615b8 2024-11-28T03:42:00,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ab304b2a3e2940b7b174abf791e615b8, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T03:42:00,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1a2c0de76d7e5c66916b7f6387695a75 in 1328ms, sequenceid=12, compaction requested=false 2024-11-28T03:42:00,844 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T03:42:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:00,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T03:42:00,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:00,890 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-28T03:42:00,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:00,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:00,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:00,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:00,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:00,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:00,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/95d4314e584c48809ec27b156c581336 is 50, key is test_row_0/A:col10/1732765319630/Put/seqid=0 2024-11-28T03:42:00,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742079_1255 (size=12001) 2024-11-28T03:42:01,311 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/95d4314e584c48809ec27b156c581336 2024-11-28T03:42:01,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/b9e30b233569428a94c4904780a99ec2 is 50, key is test_row_0/B:col10/1732765319630/Put/seqid=0 2024-11-28T03:42:01,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742080_1256 (size=12001) 2024-11-28T03:42:01,335 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/b9e30b233569428a94c4904780a99ec2 2024-11-28T03:42:01,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/7049c5c085804a5a9b8c59ff3fe7e22e is 50, key is test_row_0/C:col10/1732765319630/Put/seqid=0 2024-11-28T03:42:01,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742081_1257 (size=12001) 2024-11-28T03:42:01,356 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/7049c5c085804a5a9b8c59ff3fe7e22e 2024-11-28T03:42:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/95d4314e584c48809ec27b156c581336 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/95d4314e584c48809ec27b156c581336 2024-11-28T03:42:01,368 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/95d4314e584c48809ec27b156c581336, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T03:42:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/b9e30b233569428a94c4904780a99ec2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9e30b233569428a94c4904780a99ec2 2024-11-28T03:42:01,376 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9e30b233569428a94c4904780a99ec2, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T03:42:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/7049c5c085804a5a9b8c59ff3fe7e22e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/7049c5c085804a5a9b8c59ff3fe7e22e 2024-11-28T03:42:01,382 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/7049c5c085804a5a9b8c59ff3fe7e22e, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T03:42:01,383 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for 1a2c0de76d7e5c66916b7f6387695a75 in 493ms, sequenceid=47, compaction requested=false 2024-11-28T03:42:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:01,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-28T03:42:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-28T03:42:01,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-28T03:42:01,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-28T03:42:01,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.8940 sec 2024-11-28T03:42:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T03:42:01,647 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:01,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-28T03:42:01,649 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T03:42:01,650 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:01,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T03:42:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:01,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:42:01,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:01,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:01,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:01,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:01,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:01,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:01,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/438708826f54453eb8ad947d609a2901 is 50, key is test_row_0/A:col10/1732765321769/Put/seqid=0 2024-11-28T03:42:01,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742082_1258 (size=12001) 2024-11-28T03:42:01,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/438708826f54453eb8ad947d609a2901 2024-11-28T03:42:01,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/58445c9b179d4496bd2793f9228838a0 is 50, key is test_row_0/B:col10/1732765321769/Put/seqid=0 2024-11-28T03:42:01,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-28T03:42:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:01,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:01,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742083_1259 (size=12001) 2024-11-28T03:42:01,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/58445c9b179d4496bd2793f9228838a0 2024-11-28T03:42:01,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765321807, max timestamp=1732765321818Num uniques 859; 2024-11-28T03:42:01,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:42:01,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765321807, max timestamp=1732765321818Num uniques 859; 2024-11-28T03:42:01,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:42:01,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765321807, max timestamp=1732765321818Num uniques 859; 2024-11-28T03:42:01,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:42:01,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/557699d27d5d4846a84c1508e3532b2e is 50, key is test_row_0/C:col10/1732765321769/Put/seqid=0 2024-11-28T03:42:01,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742084_1260 (size=12001) 2024-11-28T03:42:01,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/557699d27d5d4846a84c1508e3532b2e 2024-11-28T03:42:01,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/438708826f54453eb8ad947d609a2901 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/438708826f54453eb8ad947d609a2901 2024-11-28T03:42:01,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/438708826f54453eb8ad947d609a2901, entries=150, sequenceid=57, filesize=11.7 K 2024-11-28T03:42:01,855 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-28T03:42:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765381849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765381851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:01,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/58445c9b179d4496bd2793f9228838a0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58445c9b179d4496bd2793f9228838a0 2024-11-28T03:42:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:01,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58445c9b179d4496bd2793f9228838a0, entries=150, sequenceid=57, filesize=11.7 K 2024-11-28T03:42:01,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765381853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:01,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765381855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:01,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:01,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:01,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/557699d27d5d4846a84c1508e3532b2e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/557699d27d5d4846a84c1508e3532b2e 2024-11-28T03:42:01,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765381857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:01,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/557699d27d5d4846a84c1508e3532b2e, entries=150, sequenceid=57, filesize=11.7 K 2024-11-28T03:42:01,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 1a2c0de76d7e5c66916b7f6387695a75 in 100ms, sequenceid=57, compaction requested=true 2024-11-28T03:42:01,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:01,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:01,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:01,873 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:01,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:01,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:01,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:01,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T03:42:01,874 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:01,876 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:01,876 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:01,876 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:01,876 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:01,877 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c11430cbddec4f7b8b1a0fbe92cadf80, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9e30b233569428a94c4904780a99ec2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58445c9b179d4496bd2793f9228838a0] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.2 K 2024-11-28T03:42:01,878 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:01,879 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:01,879 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/ede0dfa327c04956a6499ab590513518, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/95d4314e584c48809ec27b156c581336, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/438708826f54453eb8ad947d609a2901] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.2 K 2024-11-28T03:42:01,880 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ede0dfa327c04956a6499ab590513518, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732765319506 2024-11-28T03:42:01,880 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c11430cbddec4f7b8b1a0fbe92cadf80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732765319506 2024-11-28T03:42:01,880 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d4314e584c48809ec27b156c581336, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732765319619 2024-11-28T03:42:01,881 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e30b233569428a94c4904780a99ec2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732765319619 2024-11-28T03:42:01,881 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58445c9b179d4496bd2793f9228838a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732765321765 2024-11-28T03:42:01,881 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 438708826f54453eb8ad947d609a2901, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732765321765 2024-11-28T03:42:01,894 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:01,895 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/865916afec474c8391700cbaea3709e8 is 50, key is test_row_0/B:col10/1732765321769/Put/seqid=0 2024-11-28T03:42:01,921 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:01,922 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/e3e989e361f1413eb21e3c6f0093c34e is 50, key is test_row_0/A:col10/1732765321769/Put/seqid=0 2024-11-28T03:42:01,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-28T03:42:01,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:01,958 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-28T03:42:01,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:01,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:01,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:01,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:01,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:01,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742086_1262 (size=12104) 2024-11-28T03:42:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T03:42:01,969 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/e3e989e361f1413eb21e3c6f0093c34e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e3e989e361f1413eb21e3c6f0093c34e 2024-11-28T03:42:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:01,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:01,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742085_1261 (size=12104) 2024-11-28T03:42:01,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/1983bafceb1d4da4ab88846b8b28fd30 is 50, key is test_row_0/A:col10/1732765321852/Put/seqid=0 2024-11-28T03:42:01,976 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into e3e989e361f1413eb21e3c6f0093c34e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:01,976 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:01,976 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=13, startTime=1732765321873; duration=0sec 2024-11-28T03:42:01,977 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:01,977 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:01,977 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:01,979 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:01,979 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:01,979 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:01,979 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ab304b2a3e2940b7b174abf791e615b8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/7049c5c085804a5a9b8c59ff3fe7e22e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/557699d27d5d4846a84c1508e3532b2e] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.2 K 2024-11-28T03:42:01,980 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ab304b2a3e2940b7b174abf791e615b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732765319506 2024-11-28T03:42:01,982 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7049c5c085804a5a9b8c59ff3fe7e22e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732765319619 2024-11-28T03:42:01,983 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 557699d27d5d4846a84c1508e3532b2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732765321765 2024-11-28T03:42:01,986 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/865916afec474c8391700cbaea3709e8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/865916afec474c8391700cbaea3709e8 2024-11-28T03:42:01,996 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into 865916afec474c8391700cbaea3709e8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:01,996 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:01,996 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=13, startTime=1732765321873; duration=0sec 2024-11-28T03:42:01,996 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:01,996 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:02,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765381980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765381985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765381985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765381988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765382001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742087_1263 (size=12001) 2024-11-28T03:42:02,025 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:02,025 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/5b74a4dda976496a8f22b569a9ea6149 is 50, key is test_row_0/C:col10/1732765321769/Put/seqid=0 2024-11-28T03:42:02,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742088_1264 (size=12104) 2024-11-28T03:42:02,080 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/5b74a4dda976496a8f22b569a9ea6149 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/5b74a4dda976496a8f22b569a9ea6149 2024-11-28T03:42:02,088 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into 5b74a4dda976496a8f22b569a9ea6149(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:02,088 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:02,088 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=13, startTime=1732765321873; duration=0sec 2024-11-28T03:42:02,088 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:02,088 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:02,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765382107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765382117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765382117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765382117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765382117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T03:42:02,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765382316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765382326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765382327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765382327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765382327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,417 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/1983bafceb1d4da4ab88846b8b28fd30 2024-11-28T03:42:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/c6c5b54975864f36b8965ef60ca5ee68 is 50, key is test_row_0/B:col10/1732765321852/Put/seqid=0 2024-11-28T03:42:02,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742089_1265 (size=12001) 2024-11-28T03:42:02,483 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/c6c5b54975864f36b8965ef60ca5ee68 2024-11-28T03:42:02,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/db5e710b90be4974999099bb1a521018 is 50, key is test_row_0/C:col10/1732765321852/Put/seqid=0 2024-11-28T03:42:02,509 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:42:02,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742090_1266 (size=12001) 2024-11-28T03:42:02,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765382626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765382636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765382636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765382636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765382636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T03:42:02,979 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/db5e710b90be4974999099bb1a521018 2024-11-28T03:42:02,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/1983bafceb1d4da4ab88846b8b28fd30 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1983bafceb1d4da4ab88846b8b28fd30 2024-11-28T03:42:02,993 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1983bafceb1d4da4ab88846b8b28fd30, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T03:42:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/c6c5b54975864f36b8965ef60ca5ee68 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c6c5b54975864f36b8965ef60ca5ee68 2024-11-28T03:42:03,001 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c6c5b54975864f36b8965ef60ca5ee68, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T03:42:03,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/db5e710b90be4974999099bb1a521018 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/db5e710b90be4974999099bb1a521018 2024-11-28T03:42:03,020 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/db5e710b90be4974999099bb1a521018, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T03:42:03,021 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 1a2c0de76d7e5c66916b7f6387695a75 in 1063ms, sequenceid=94, compaction requested=false 2024-11-28T03:42:03,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:03,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:03,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-28T03:42:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-28T03:42:03,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-28T03:42:03,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3720 sec 2024-11-28T03:42:03,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.3810 sec 2024-11-28T03:42:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:03,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:42:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:03,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/e87a56b9725247ab8a7f7a1cf21a9a27 is 50, key is test_row_0/A:col10/1732765321985/Put/seqid=0 2024-11-28T03:42:03,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742091_1267 (size=12001) 2024-11-28T03:42:03,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765323185, max timestamp=1732765323205Num uniques 859; 2024-11-28T03:42:03,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:03,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765323185, max timestamp=1732765323205Num uniques 859; 2024-11-28T03:42:03,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:03,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765323185, max timestamp=1732765323205Num uniques 859; 2024-11-28T03:42:03,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:03,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765383234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765383236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765383240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765383242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765383243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765383346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765383346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765383357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765383357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765383357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765383556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765383566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765383566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765383567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765383567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/e87a56b9725247ab8a7f7a1cf21a9a27 2024-11-28T03:42:03,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/8519e9db8dc3439db3be13095a49f666 is 50, key is test_row_0/B:col10/1732765321985/Put/seqid=0 2024-11-28T03:42:03,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742092_1268 (size=12001) 2024-11-28T03:42:03,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/8519e9db8dc3439db3be13095a49f666 2024-11-28T03:42:03,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/3b32bdd5f26f4cb0900dcdd27e58f18b is 50, key is test_row_0/C:col10/1732765321985/Put/seqid=0 2024-11-28T03:42:03,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742093_1269 (size=12001) 2024-11-28T03:42:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T03:42:03,797 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:03,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-28T03:42:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T03:42:03,800 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:03,801 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:03,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:03,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765383866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765383876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765383877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765383877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765383877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T03:42:03,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T03:42:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:03,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:04,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/3b32bdd5f26f4cb0900dcdd27e58f18b 2024-11-28T03:42:04,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/e87a56b9725247ab8a7f7a1cf21a9a27 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e87a56b9725247ab8a7f7a1cf21a9a27 2024-11-28T03:42:04,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e87a56b9725247ab8a7f7a1cf21a9a27, entries=150, sequenceid=107, filesize=11.7 K 2024-11-28T03:42:04,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/8519e9db8dc3439db3be13095a49f666 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8519e9db8dc3439db3be13095a49f666 2024-11-28T03:42:04,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8519e9db8dc3439db3be13095a49f666, entries=150, sequenceid=107, filesize=11.7 K 2024-11-28T03:42:04,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/3b32bdd5f26f4cb0900dcdd27e58f18b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3b32bdd5f26f4cb0900dcdd27e58f18b 2024-11-28T03:42:04,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3b32bdd5f26f4cb0900dcdd27e58f18b, entries=150, sequenceid=107, filesize=11.7 K 2024-11-28T03:42:04,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1a2c0de76d7e5c66916b7f6387695a75 in 892ms, sequenceid=107, compaction requested=true 2024-11-28T03:42:04,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:04,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:04,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:04,041 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:04,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:04,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:04,041 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:04,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:04,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:04,045 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:04,045 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:04,045 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:04,045 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:04,045 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:04,045 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:04,045 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e3e989e361f1413eb21e3c6f0093c34e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1983bafceb1d4da4ab88846b8b28fd30, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e87a56b9725247ab8a7f7a1cf21a9a27] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.3 K 2024-11-28T03:42:04,045 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/865916afec474c8391700cbaea3709e8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c6c5b54975864f36b8965ef60ca5ee68, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8519e9db8dc3439db3be13095a49f666] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.3 K 2024-11-28T03:42:04,045 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3e989e361f1413eb21e3c6f0093c34e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732765321765 2024-11-28T03:42:04,045 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 865916afec474c8391700cbaea3709e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732765321765 2024-11-28T03:42:04,046 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1983bafceb1d4da4ab88846b8b28fd30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732765321850 2024-11-28T03:42:04,046 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c5b54975864f36b8965ef60ca5ee68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732765321850 2024-11-28T03:42:04,046 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e87a56b9725247ab8a7f7a1cf21a9a27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732765321982 2024-11-28T03:42:04,046 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8519e9db8dc3439db3be13095a49f666, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732765321982 2024-11-28T03:42:04,062 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:04,063 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:04,063 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/b9db127ba09e4df5b5ec9975b0ad4293 is 50, key is test_row_0/B:col10/1732765321985/Put/seqid=0 2024-11-28T03:42:04,063 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/9a1f1740bcfe4313aa927725983b2c26 is 50, key is test_row_0/A:col10/1732765321985/Put/seqid=0 2024-11-28T03:42:04,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742095_1271 (size=12207) 2024-11-28T03:42:04,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742094_1270 (size=12207) 2024-11-28T03:42:04,085 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/9a1f1740bcfe4313aa927725983b2c26 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9a1f1740bcfe4313aa927725983b2c26 2024-11-28T03:42:04,091 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into 9a1f1740bcfe4313aa927725983b2c26(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:04,091 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:04,091 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=13, startTime=1732765324041; duration=0sec 2024-11-28T03:42:04,091 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:04,091 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:04,091 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:04,092 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:04,093 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:04,093 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:04,093 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/5b74a4dda976496a8f22b569a9ea6149, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/db5e710b90be4974999099bb1a521018, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3b32bdd5f26f4cb0900dcdd27e58f18b] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.3 K 2024-11-28T03:42:04,093 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b74a4dda976496a8f22b569a9ea6149, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732765321765 2024-11-28T03:42:04,094 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting db5e710b90be4974999099bb1a521018, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732765321850 2024-11-28T03:42:04,094 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b32bdd5f26f4cb0900dcdd27e58f18b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732765321982 2024-11-28T03:42:04,103 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:04,103 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/16f0534246d647d18bc95a4e95e63ec9 is 50, key is test_row_0/C:col10/1732765321985/Put/seqid=0 2024-11-28T03:42:04,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T03:42:04,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:04,108 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T03:42:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T03:42:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/2ca894b13f4e41afa75dbbfb8d9097d4 is 50, key is test_row_0/A:col10/1732765323241/Put/seqid=0 2024-11-28T03:42:04,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742096_1272 (size=12207) 2024-11-28T03:42:04,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742097_1273 (size=12151) 2024-11-28T03:42:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:04,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:04,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765384397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765384397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765384398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765384399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765384402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T03:42:04,488 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/b9db127ba09e4df5b5ec9975b0ad4293 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9db127ba09e4df5b5ec9975b0ad4293 2024-11-28T03:42:04,494 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into b9db127ba09e4df5b5ec9975b0ad4293(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:04,494 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:04,494 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=13, startTime=1732765324041; duration=0sec 2024-11-28T03:42:04,494 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:04,494 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:04,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765384506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765384506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765384517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,536 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/16f0534246d647d18bc95a4e95e63ec9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/16f0534246d647d18bc95a4e95e63ec9 2024-11-28T03:42:04,538 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/2ca894b13f4e41afa75dbbfb8d9097d4 2024-11-28T03:42:04,544 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into 16f0534246d647d18bc95a4e95e63ec9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:04,544 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:04,544 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=13, startTime=1732765324041; duration=0sec 2024-11-28T03:42:04,544 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:04,544 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:04,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/74fab4430c2c4adfac77fb47dd1ddd88 is 50, key is test_row_0/B:col10/1732765323241/Put/seqid=0 2024-11-28T03:42:04,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742098_1274 (size=12151) 2024-11-28T03:42:04,558 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/74fab4430c2c4adfac77fb47dd1ddd88 2024-11-28T03:42:04,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/770b960c02ab4b76886ccc807aedd8f8 is 50, key is test_row_0/C:col10/1732765323241/Put/seqid=0 2024-11-28T03:42:04,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742099_1275 (size=12151) 2024-11-28T03:42:04,573 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/770b960c02ab4b76886ccc807aedd8f8 2024-11-28T03:42:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/2ca894b13f4e41afa75dbbfb8d9097d4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/2ca894b13f4e41afa75dbbfb8d9097d4 2024-11-28T03:42:04,582 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/2ca894b13f4e41afa75dbbfb8d9097d4, entries=150, sequenceid=145, filesize=11.9 K 2024-11-28T03:42:04,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/74fab4430c2c4adfac77fb47dd1ddd88 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74fab4430c2c4adfac77fb47dd1ddd88 2024-11-28T03:42:04,589 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74fab4430c2c4adfac77fb47dd1ddd88, entries=150, sequenceid=145, filesize=11.9 K 2024-11-28T03:42:04,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/770b960c02ab4b76886ccc807aedd8f8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/770b960c02ab4b76886ccc807aedd8f8 2024-11-28T03:42:04,597 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/770b960c02ab4b76886ccc807aedd8f8, entries=150, sequenceid=145, filesize=11.9 K 2024-11-28T03:42:04,598 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 1a2c0de76d7e5c66916b7f6387695a75 in 490ms, sequenceid=145, compaction requested=false 2024-11-28T03:42:04,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:04,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:04,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-28T03:42:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-28T03:42:04,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-28T03:42:04,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 799 msec 2024-11-28T03:42:04,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 806 msec 2024-11-28T03:42:04,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:04,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T03:42:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:04,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:04,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:04,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:04,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/07e82603db8547eda7f112fad6c5fed4 is 50, key is test_row_0/A:col10/1732765324719/Put/seqid=0 2024-11-28T03:42:04,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742100_1276 (size=12151) 2024-11-28T03:42:04,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/07e82603db8547eda7f112fad6c5fed4 2024-11-28T03:42:04,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765384760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732765324749, max timestamp=1732765324759Num uniques 859; 2024-11-28T03:42:04,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:04,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765324749, max timestamp=1732765324759Num uniques 859; 2024-11-28T03:42:04,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:04,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732765324751, max timestamp=1732765324759Num uniques 859; 2024-11-28T03:42:04,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:42:04,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/83720ad7a3444cacae4a217c9d72e1ca is 50, key is test_row_0/B:col10/1732765324719/Put/seqid=0 2024-11-28T03:42:04,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742101_1277 (size=12151) 2024-11-28T03:42:04,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/83720ad7a3444cacae4a217c9d72e1ca 2024-11-28T03:42:04,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/9f3974ba809940da90bc4a083ed1621f is 50, key is test_row_0/C:col10/1732765324719/Put/seqid=0 2024-11-28T03:42:04,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742102_1278 (size=12151) 2024-11-28T03:42:04,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765384807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765384810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765384876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765384917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765384926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:04,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T03:42:04,936 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:04,937 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:04,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-28T03:42:04,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T03:42:04,940 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:04,940 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:04,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T03:42:05,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765385086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T03:42:05,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:05,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:05,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:05,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:05,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:05,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765385127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765385136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/9f3974ba809940da90bc4a083ed1621f 2024-11-28T03:42:05,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/07e82603db8547eda7f112fad6c5fed4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/07e82603db8547eda7f112fad6c5fed4 2024-11-28T03:42:05,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/07e82603db8547eda7f112fad6c5fed4, entries=150, sequenceid=159, filesize=11.9 K 2024-11-28T03:42:05,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/83720ad7a3444cacae4a217c9d72e1ca as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/83720ad7a3444cacae4a217c9d72e1ca 2024-11-28T03:42:05,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/83720ad7a3444cacae4a217c9d72e1ca, entries=150, sequenceid=159, filesize=11.9 K 2024-11-28T03:42:05,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/9f3974ba809940da90bc4a083ed1621f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9f3974ba809940da90bc4a083ed1621f 2024-11-28T03:42:05,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9f3974ba809940da90bc4a083ed1621f, entries=150, sequenceid=159, filesize=11.9 K 2024-11-28T03:42:05,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=125.23 KB/128240 for 1a2c0de76d7e5c66916b7f6387695a75 in 504ms, sequenceid=159, compaction requested=true 2024-11-28T03:42:05,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:05,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:05,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:05,223 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:05,223 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:05,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:05,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:05,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:05,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:05,224 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:05,224 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:05,224 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:05,224 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:05,224 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:05,224 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:05,224 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9a1f1740bcfe4313aa927725983b2c26, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/2ca894b13f4e41afa75dbbfb8d9097d4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/07e82603db8547eda7f112fad6c5fed4] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.7 K 2024-11-28T03:42:05,224 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9db127ba09e4df5b5ec9975b0ad4293, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74fab4430c2c4adfac77fb47dd1ddd88, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/83720ad7a3444cacae4a217c9d72e1ca] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.7 K 2024-11-28T03:42:05,225 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting b9db127ba09e4df5b5ec9975b0ad4293, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732765321982 2024-11-28T03:42:05,225 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a1f1740bcfe4313aa927725983b2c26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732765321982 2024-11-28T03:42:05,226 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 74fab4430c2c4adfac77fb47dd1ddd88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732765323233 2024-11-28T03:42:05,226 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ca894b13f4e41afa75dbbfb8d9097d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732765323233 2024-11-28T03:42:05,226 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 83720ad7a3444cacae4a217c9d72e1ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732765324390 2024-11-28T03:42:05,226 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e82603db8547eda7f112fad6c5fed4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732765324390 2024-11-28T03:42:05,233 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:05,234 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/c5a7f7cdabbb40fe9de3e43ab1660cab is 50, key is test_row_0/B:col10/1732765324719/Put/seqid=0 2024-11-28T03:42:05,236 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:05,237 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/92c54c0d929f4f88b3e5c0af418c9a21 is 50, key is test_row_0/A:col10/1732765324719/Put/seqid=0 2024-11-28T03:42:05,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742104_1280 (size=12459) 2024-11-28T03:42:05,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742103_1279 (size=12459) 2024-11-28T03:42:05,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T03:42:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:05,248 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=125.23 KB heapSize=350.61 KB 2024-11-28T03:42:05,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:05,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:05,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:05,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:05,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:05,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:05,249 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/c5a7f7cdabbb40fe9de3e43ab1660cab as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c5a7f7cdabbb40fe9de3e43ab1660cab 2024-11-28T03:42:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T03:42:05,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/9d6eeaf247ad4564a4dfbf16176c083e is 50, key is test_row_0/A:col10/1732765324797/Put/seqid=0 2024-11-28T03:42:05,297 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into c5a7f7cdabbb40fe9de3e43ab1660cab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:05,298 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:05,298 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=13, startTime=1732765325223; duration=0sec 2024-11-28T03:42:05,298 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:05,298 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:05,298 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:05,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742105_1281 (size=12151) 2024-11-28T03:42:05,302 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:05,303 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:05,303 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:05,303 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/16f0534246d647d18bc95a4e95e63ec9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/770b960c02ab4b76886ccc807aedd8f8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9f3974ba809940da90bc4a083ed1621f] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.7 K 2024-11-28T03:42:05,303 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 16f0534246d647d18bc95a4e95e63ec9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732765321982 2024-11-28T03:42:05,304 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 770b960c02ab4b76886ccc807aedd8f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732765323233 2024-11-28T03:42:05,304 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3974ba809940da90bc4a083ed1621f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732765324390 2024-11-28T03:42:05,320 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:05,321 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/5792ce52a136421fa834cb8da2e02e6b is 50, key is test_row_0/C:col10/1732765324719/Put/seqid=0 2024-11-28T03:42:05,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742106_1282 (size=12459) 2024-11-28T03:42:05,338 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/5792ce52a136421fa834cb8da2e02e6b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/5792ce52a136421fa834cb8da2e02e6b 2024-11-28T03:42:05,347 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into 5792ce52a136421fa834cb8da2e02e6b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:05,347 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:05,348 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=13, startTime=1732765325223; duration=0sec 2024-11-28T03:42:05,348 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:05,348 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:05,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:05,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765385425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765385425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765385426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765385436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765385447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765385537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765385537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765385537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T03:42:05,651 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/92c54c0d929f4f88b3e5c0af418c9a21 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/92c54c0d929f4f88b3e5c0af418c9a21 2024-11-28T03:42:05,656 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into 92c54c0d929f4f88b3e5c0af418c9a21(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:05,656 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:05,656 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=13, startTime=1732765325223; duration=0sec 2024-11-28T03:42:05,657 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:05,657 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:05,702 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/9d6eeaf247ad4564a4dfbf16176c083e 2024-11-28T03:42:05,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/36688c827c614181811d077b16ba8540 is 50, key is test_row_0/B:col10/1732765324797/Put/seqid=0 2024-11-28T03:42:05,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742107_1283 (size=12151) 2024-11-28T03:42:05,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765385747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765385747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765385747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765385957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765385957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765386057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765386057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765386057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T03:42:06,116 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/36688c827c614181811d077b16ba8540 2024-11-28T03:42:06,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/9de09106a95c41899a567894af2228d8 is 50, key is test_row_0/C:col10/1732765324797/Put/seqid=0 2024-11-28T03:42:06,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742108_1284 (size=12151) 2024-11-28T03:42:06,532 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/9de09106a95c41899a567894af2228d8 2024-11-28T03:42:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/9d6eeaf247ad4564a4dfbf16176c083e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d6eeaf247ad4564a4dfbf16176c083e 2024-11-28T03:42:06,542 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d6eeaf247ad4564a4dfbf16176c083e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-28T03:42:06,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/36688c827c614181811d077b16ba8540 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/36688c827c614181811d077b16ba8540 2024-11-28T03:42:06,547 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/36688c827c614181811d077b16ba8540, entries=150, sequenceid=193, filesize=11.9 K 2024-11-28T03:42:06,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/9de09106a95c41899a567894af2228d8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9de09106a95c41899a567894af2228d8 2024-11-28T03:42:06,552 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9de09106a95c41899a567894af2228d8, entries=150, sequenceid=193, filesize=11.9 K 2024-11-28T03:42:06,553 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~350.56 KB/358976, currentSize=60.38 KB/61830 for 1a2c0de76d7e5c66916b7f6387695a75 in 1306ms, sequenceid=193, compaction requested=false 2024-11-28T03:42:06,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:06,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:06,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-28T03:42:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-28T03:42:06,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-28T03:42:06,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6130 sec 2024-11-28T03:42:06,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6200 sec 2024-11-28T03:42:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:06,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T03:42:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:06,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6e7888e4d5954f71a5b6efb2255f2237 is 50, key is test_row_0/A:col10/1732765325416/Put/seqid=0 2024-11-28T03:42:06,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742109_1285 (size=14541) 2024-11-28T03:42:06,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765386597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765386600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765386602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765386706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765386716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765386716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765386916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765386926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765386926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765386966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,970 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_0' in 1a2c0de76d7e5c66916b7f6387695a75 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765386976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:06,980 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in 1a2c0de76d7e5c66916b7f6387695a75 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:06,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6e7888e4d5954f71a5b6efb2255f2237 2024-11-28T03:42:06,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/74b4041c533c44aeb5fa2e305975e2d4 is 50, key is test_row_0/B:col10/1732765325416/Put/seqid=0 2024-11-28T03:42:06,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742110_1286 (size=12151) 2024-11-28T03:42:06,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/74b4041c533c44aeb5fa2e305975e2d4 2024-11-28T03:42:07,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/664722a7215d4e5591a103eb8f5d5b01 is 50, key is test_row_0/C:col10/1732765325416/Put/seqid=0 2024-11-28T03:42:07,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742111_1287 (size=12151) 2024-11-28T03:42:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T03:42:07,097 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:07,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-28T03:42:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T03:42:07,099 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:07,100 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:07,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T03:42:07,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765387226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:07,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765387236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:07,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765387236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:07,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T03:42:07,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:07,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:07,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:07,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/664722a7215d4e5591a103eb8f5d5b01 2024-11-28T03:42:07,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T03:42:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:07,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6e7888e4d5954f71a5b6efb2255f2237 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6e7888e4d5954f71a5b6efb2255f2237 2024-11-28T03:42:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T03:42:07,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6e7888e4d5954f71a5b6efb2255f2237, entries=200, sequenceid=210, filesize=14.2 K 2024-11-28T03:42:07,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/74b4041c533c44aeb5fa2e305975e2d4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74b4041c533c44aeb5fa2e305975e2d4 2024-11-28T03:42:07,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74b4041c533c44aeb5fa2e305975e2d4, entries=150, sequenceid=210, filesize=11.9 K 2024-11-28T03:42:07,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/664722a7215d4e5591a103eb8f5d5b01 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/664722a7215d4e5591a103eb8f5d5b01 2024-11-28T03:42:07,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/664722a7215d4e5591a103eb8f5d5b01, entries=150, sequenceid=210, filesize=11.9 K 2024-11-28T03:42:07,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1a2c0de76d7e5c66916b7f6387695a75 in 860ms, sequenceid=210, compaction requested=true 2024-11-28T03:42:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:07,428 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:07,428 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:07,429 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:07,429 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:07,429 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:07,429 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:07,430 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,430 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,430 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/92c54c0d929f4f88b3e5c0af418c9a21, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d6eeaf247ad4564a4dfbf16176c083e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6e7888e4d5954f71a5b6efb2255f2237] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=38.2 K 2024-11-28T03:42:07,430 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c5a7f7cdabbb40fe9de3e43ab1660cab, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/36688c827c614181811d077b16ba8540, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74b4041c533c44aeb5fa2e305975e2d4] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.9 K 2024-11-28T03:42:07,430 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c54c0d929f4f88b3e5c0af418c9a21, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732765324390 2024-11-28T03:42:07,430 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting c5a7f7cdabbb40fe9de3e43ab1660cab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732765324390 2024-11-28T03:42:07,430 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d6eeaf247ad4564a4dfbf16176c083e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732765324797 2024-11-28T03:42:07,430 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 36688c827c614181811d077b16ba8540, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732765324797 2024-11-28T03:42:07,431 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 74b4041c533c44aeb5fa2e305975e2d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732765325416 2024-11-28T03:42:07,431 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e7888e4d5954f71a5b6efb2255f2237, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732765325416 2024-11-28T03:42:07,440 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:07,441 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/f57ae1123a384198ad10f35b52972322 is 50, key is test_row_0/B:col10/1732765325416/Put/seqid=0 2024-11-28T03:42:07,443 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:07,444 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/638993eef20c422b9798b510e48dfc07 is 50, key is test_row_0/A:col10/1732765325416/Put/seqid=0 2024-11-28T03:42:07,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742112_1288 (size=12561) 2024-11-28T03:42:07,459 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/f57ae1123a384198ad10f35b52972322 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f57ae1123a384198ad10f35b52972322 2024-11-28T03:42:07,465 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into f57ae1123a384198ad10f35b52972322(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:07,465 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:07,465 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=13, startTime=1732765327428; duration=0sec 2024-11-28T03:42:07,465 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:07,465 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:07,465 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:07,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742113_1289 (size=12561) 2024-11-28T03:42:07,468 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:07,468 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:07,468 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,469 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/5792ce52a136421fa834cb8da2e02e6b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9de09106a95c41899a567894af2228d8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/664722a7215d4e5591a103eb8f5d5b01] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=35.9 K 2024-11-28T03:42:07,469 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5792ce52a136421fa834cb8da2e02e6b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732765324390 2024-11-28T03:42:07,469 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de09106a95c41899a567894af2228d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732765324797 2024-11-28T03:42:07,470 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 664722a7215d4e5591a103eb8f5d5b01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732765325416 2024-11-28T03:42:07,475 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/638993eef20c422b9798b510e48dfc07 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/638993eef20c422b9798b510e48dfc07 2024-11-28T03:42:07,482 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:07,482 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/d652de2c3f1e4d54b61c85725d3951d8 is 50, key is test_row_0/C:col10/1732765325416/Put/seqid=0 2024-11-28T03:42:07,483 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into 638993eef20c422b9798b510e48dfc07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:07,484 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:07,484 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=13, startTime=1732765327428; duration=0sec 2024-11-28T03:42:07,484 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:07,484 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:07,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742114_1290 (size=12561) 2024-11-28T03:42:07,501 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/d652de2c3f1e4d54b61c85725d3951d8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/d652de2c3f1e4d54b61c85725d3951d8 2024-11-28T03:42:07,510 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into d652de2c3f1e4d54b61c85725d3951d8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:07,510 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:07,510 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=13, startTime=1732765327428; duration=0sec 2024-11-28T03:42:07,510 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:07,510 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:07,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T03:42:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,561 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:42:07,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:07,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:07,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:07,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:07,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:07,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:07,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/a38974465307498c9a13e705769cca3a is 50, key is test_row_0/A:col10/1732765326600/Put/seqid=0 2024-11-28T03:42:07,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742115_1291 (size=12151) 2024-11-28T03:42:07,573 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/a38974465307498c9a13e705769cca3a 2024-11-28T03:42:07,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/3cb4cf3ff88b42a9b9e2b004080f58c5 is 50, key is test_row_0/B:col10/1732765326600/Put/seqid=0 2024-11-28T03:42:07,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742116_1292 (size=12151) 2024-11-28T03:42:07,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/3cb4cf3ff88b42a9b9e2b004080f58c5 2024-11-28T03:42:07,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/9fddd83d4cc94e3cba2cb4147971ea58 is 50, key is test_row_0/C:col10/1732765326600/Put/seqid=0 2024-11-28T03:42:07,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742117_1293 (size=12151) 2024-11-28T03:42:07,658 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/9fddd83d4cc94e3cba2cb4147971ea58 2024-11-28T03:42:07,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/a38974465307498c9a13e705769cca3a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/a38974465307498c9a13e705769cca3a 2024-11-28T03:42:07,676 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/a38974465307498c9a13e705769cca3a, entries=150, sequenceid=231, filesize=11.9 K 2024-11-28T03:42:07,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/3cb4cf3ff88b42a9b9e2b004080f58c5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3cb4cf3ff88b42a9b9e2b004080f58c5 2024-11-28T03:42:07,683 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3cb4cf3ff88b42a9b9e2b004080f58c5, entries=150, sequenceid=231, filesize=11.9 K 2024-11-28T03:42:07,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/9fddd83d4cc94e3cba2cb4147971ea58 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9fddd83d4cc94e3cba2cb4147971ea58 2024-11-28T03:42:07,689 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9fddd83d4cc94e3cba2cb4147971ea58, entries=150, sequenceid=231, filesize=11.9 K 2024-11-28T03:42:07,690 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 1a2c0de76d7e5c66916b7f6387695a75 in 129ms, sequenceid=231, compaction requested=false 2024-11-28T03:42:07,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:07,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-28T03:42:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-28T03:42:07,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-28T03:42:07,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 592 msec 2024-11-28T03:42:07,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 598 msec 2024-11-28T03:42:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T03:42:07,727 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:07,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-28T03:42:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T03:42:07,729 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:07,731 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:07,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:07,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:42:07,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:07,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:07,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:07,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:07,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:07,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:07,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/aff0cd7e153a41f4be55f37a1f62ea6c is 50, key is test_row_0/A:col10/1732765327750/Put/seqid=0 2024-11-28T03:42:07,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742118_1294 (size=12147) 2024-11-28T03:42:07,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/aff0cd7e153a41f4be55f37a1f62ea6c 2024-11-28T03:42:07,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/bc722f1851b5441aa667ccf193ff1a17 is 50, key is test_row_0/B:col10/1732765327750/Put/seqid=0 2024-11-28T03:42:07,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765327771, max timestamp=1732765327790Num uniques 859; 2024-11-28T03:42:07,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765327771, max timestamp=1732765327790Num uniques 859; 2024-11-28T03:42:07,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:07,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:07,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765327771, max timestamp=1732765327790Num uniques 859; 2024-11-28T03:42:07,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:07,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742119_1295 (size=9757) 2024-11-28T03:42:07,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/bc722f1851b5441aa667ccf193ff1a17 2024-11-28T03:42:07,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/813b9eefc43145ec9822db1cae4a7cd3 is 50, key is test_row_0/C:col10/1732765327750/Put/seqid=0 2024-11-28T03:42:07,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742120_1296 (size=9757) 2024-11-28T03:42:07,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/813b9eefc43145ec9822db1cae4a7cd3 2024-11-28T03:42:07,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/aff0cd7e153a41f4be55f37a1f62ea6c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/aff0cd7e153a41f4be55f37a1f62ea6c 2024-11-28T03:42:07,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/aff0cd7e153a41f4be55f37a1f62ea6c, entries=150, sequenceid=241, filesize=11.9 K 2024-11-28T03:42:07,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/bc722f1851b5441aa667ccf193ff1a17 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/bc722f1851b5441aa667ccf193ff1a17 2024-11-28T03:42:07,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/bc722f1851b5441aa667ccf193ff1a17, entries=100, sequenceid=241, filesize=9.5 K 2024-11-28T03:42:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/813b9eefc43145ec9822db1cae4a7cd3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/813b9eefc43145ec9822db1cae4a7cd3 2024-11-28T03:42:07,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765387826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:07,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/813b9eefc43145ec9822db1cae4a7cd3, entries=100, sequenceid=241, filesize=9.5 K 2024-11-28T03:42:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T03:42:07,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765387829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:07,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1a2c0de76d7e5c66916b7f6387695a75 in 85ms, sequenceid=241, compaction requested=true 2024-11-28T03:42:07,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:07,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:07,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:07,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:07,838 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:07,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:07,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:07,838 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:07,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:07,839 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36859 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:07,839 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:07,839 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:07,839 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:07,839 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,839 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,839 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/638993eef20c422b9798b510e48dfc07, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/a38974465307498c9a13e705769cca3a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/aff0cd7e153a41f4be55f37a1f62ea6c] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.0 K 2024-11-28T03:42:07,839 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f57ae1123a384198ad10f35b52972322, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3cb4cf3ff88b42a9b9e2b004080f58c5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/bc722f1851b5441aa667ccf193ff1a17] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=33.7 K 2024-11-28T03:42:07,840 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 638993eef20c422b9798b510e48dfc07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732765325416 2024-11-28T03:42:07,840 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting f57ae1123a384198ad10f35b52972322, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732765325416 2024-11-28T03:42:07,841 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a38974465307498c9a13e705769cca3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732765326596 2024-11-28T03:42:07,841 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cb4cf3ff88b42a9b9e2b004080f58c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732765326596 2024-11-28T03:42:07,842 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting aff0cd7e153a41f4be55f37a1f62ea6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732765327746 2024-11-28T03:42:07,842 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting bc722f1851b5441aa667ccf193ff1a17, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732765327746 2024-11-28T03:42:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:07,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-28T03:42:07,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:07,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:07,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:07,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:07,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:07,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:07,854 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:07,854 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/9f5b85a788cd4c2696ebc5450725f9d2 is 50, key is test_row_0/A:col10/1732765327750/Put/seqid=0 2024-11-28T03:42:07,857 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:07,857 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/3184713b908843d29aa1ea177058d620 is 50, key is test_row_0/B:col10/1732765327750/Put/seqid=0 2024-11-28T03:42:07,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/8cc1613699a0496cb24837daff198152 is 50, key is test_row_0/A:col10/1732765327822/Put/seqid=0 2024-11-28T03:42:07,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742121_1297 (size=12663) 2024-11-28T03:42:07,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:07,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765387887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:07,896 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/3184713b908843d29aa1ea177058d620 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3184713b908843d29aa1ea177058d620 2024-11-28T03:42:07,902 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into 3184713b908843d29aa1ea177058d620(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:07,902 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:07,902 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=13, startTime=1732765327838; duration=0sec 2024-11-28T03:42:07,902 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:07,902 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:07,902 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:07,904 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:07,904 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:07,905 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:07,905 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/d652de2c3f1e4d54b61c85725d3951d8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9fddd83d4cc94e3cba2cb4147971ea58, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/813b9eefc43145ec9822db1cae4a7cd3] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=33.7 K 2024-11-28T03:42:07,905 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d652de2c3f1e4d54b61c85725d3951d8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732765325416 2024-11-28T03:42:07,906 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fddd83d4cc94e3cba2cb4147971ea58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732765326596 2024-11-28T03:42:07,906 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 813b9eefc43145ec9822db1cae4a7cd3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732765327746 2024-11-28T03:42:07,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742122_1298 (size=12663) 2024-11-28T03:42:07,919 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:07,920 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/c95b538f65984809918e52a5ac6955ba is 50, key is test_row_0/C:col10/1732765327750/Put/seqid=0 2024-11-28T03:42:07,925 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/9f5b85a788cd4c2696ebc5450725f9d2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9f5b85a788cd4c2696ebc5450725f9d2 2024-11-28T03:42:07,932 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into 9f5b85a788cd4c2696ebc5450725f9d2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:07,932 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:07,932 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=13, startTime=1732765327837; duration=0sec 2024-11-28T03:42:07,933 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:07,933 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:07,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742123_1299 (size=14741) 2024-11-28T03:42:07,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/8cc1613699a0496cb24837daff198152 2024-11-28T03:42:07,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765387936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:07,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:07,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765387946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:07,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1226dd853da04f018360022c2601d633 is 50, key is test_row_0/B:col10/1732765327822/Put/seqid=0 2024-11-28T03:42:07,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742124_1300 (size=12663) 2024-11-28T03:42:07,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742125_1301 (size=12301) 2024-11-28T03:42:07,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1226dd853da04f018360022c2601d633 2024-11-28T03:42:07,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/44abd3f9652c4f32990b14c30199d5e0 is 50, key is test_row_0/C:col10/1732765327822/Put/seqid=0 2024-11-28T03:42:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765387996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742126_1302 (size=12301) 2024-11-28T03:42:08,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:08,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T03:42:08,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765388146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765388157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:08,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:08,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765388207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T03:42:08,367 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/c95b538f65984809918e52a5ac6955ba as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/c95b538f65984809918e52a5ac6955ba 2024-11-28T03:42:08,374 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into c95b538f65984809918e52a5ac6955ba(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:08,374 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:08,375 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=13, startTime=1732765327838; duration=0sec 2024-11-28T03:42:08,375 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:08,375 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:08,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/44abd3f9652c4f32990b14c30199d5e0 2024-11-28T03:42:08,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/8cc1613699a0496cb24837daff198152 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/8cc1613699a0496cb24837daff198152 2024-11-28T03:42:08,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/8cc1613699a0496cb24837daff198152, entries=200, sequenceid=278, filesize=14.4 K 2024-11-28T03:42:08,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1226dd853da04f018360022c2601d633 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1226dd853da04f018360022c2601d633 2024-11-28T03:42:08,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1226dd853da04f018360022c2601d633, entries=150, sequenceid=278, filesize=12.0 K 2024-11-28T03:42:08,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/44abd3f9652c4f32990b14c30199d5e0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/44abd3f9652c4f32990b14c30199d5e0 2024-11-28T03:42:08,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/44abd3f9652c4f32990b14c30199d5e0, entries=150, sequenceid=278, filesize=12.0 K 2024-11-28T03:42:08,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 1a2c0de76d7e5c66916b7f6387695a75 in 586ms, sequenceid=278, compaction requested=false 2024-11-28T03:42:08,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:08,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:42:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:08,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/c75146c90fc64d3f9a554fa7021b1c59 is 50, key is test_row_0/A:col10/1732765327883/Put/seqid=0 2024-11-28T03:42:08,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742127_1303 (size=12301) 2024-11-28T03:42:08,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/c75146c90fc64d3f9a554fa7021b1c59 2024-11-28T03:42:08,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/ec9ce3cfb0634abfb848bef103e2cdba is 50, key is test_row_0/B:col10/1732765327883/Put/seqid=0 2024-11-28T03:42:08,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742128_1304 (size=12301) 2024-11-28T03:42:08,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765328519, max timestamp=1732765328528Num uniques 859; 2024-11-28T03:42:08,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:42:08,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765328519, max timestamp=1732765328528Num uniques 859; 2024-11-28T03:42:08,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:42:08,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765328519, max timestamp=1732765328528Num uniques 859; 2024-11-28T03:42:08,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:42:08,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765388561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765388563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765388565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765388676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765388676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765388676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:08,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:08,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T03:42:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765388886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765388886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765388886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/ec9ce3cfb0634abfb848bef103e2cdba 2024-11-28T03:42:08,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/b9d81f0b1b1c4851a118a9111d76635b is 50, key is test_row_0/C:col10/1732765327883/Put/seqid=0 2024-11-28T03:42:08,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742129_1305 (size=12301) 2024-11-28T03:42:08,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:08,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:08,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:08,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:08,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765388986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765388986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:08,992 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_0' in 1a2c0de76d7e5c66916b7f6387695a75 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:08,992 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in 1a2c0de76d7e5c66916b7f6387695a75 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:09,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:09,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:09,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:09,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:09,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:09,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:09,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765389196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765389196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765389196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:09,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:09,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:09,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:09,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:09,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:09,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:09,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/b9d81f0b1b1c4851a118a9111d76635b 2024-11-28T03:42:09,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/c75146c90fc64d3f9a554fa7021b1c59 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/c75146c90fc64d3f9a554fa7021b1c59 2024-11-28T03:42:09,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/c75146c90fc64d3f9a554fa7021b1c59, entries=150, sequenceid=291, filesize=12.0 K 2024-11-28T03:42:09,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/ec9ce3cfb0634abfb848bef103e2cdba as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/ec9ce3cfb0634abfb848bef103e2cdba 2024-11-28T03:42:09,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/ec9ce3cfb0634abfb848bef103e2cdba, entries=150, sequenceid=291, filesize=12.0 K 2024-11-28T03:42:09,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/b9d81f0b1b1c4851a118a9111d76635b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b9d81f0b1b1c4851a118a9111d76635b 2024-11-28T03:42:09,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b9d81f0b1b1c4851a118a9111d76635b, entries=150, sequenceid=291, filesize=12.0 K 2024-11-28T03:42:09,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1a2c0de76d7e5c66916b7f6387695a75 in 902ms, sequenceid=291, compaction requested=true 2024-11-28T03:42:09,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:09,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:09,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:09,363 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:09,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:09,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:09,363 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:09,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:09,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:09,364 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:09,364 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:09,364 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:09,364 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3184713b908843d29aa1ea177058d620, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1226dd853da04f018360022c2601d633, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/ec9ce3cfb0634abfb848bef103e2cdba] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.4 K 2024-11-28T03:42:09,365 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:09,365 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:09,365 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:09,365 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9f5b85a788cd4c2696ebc5450725f9d2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/8cc1613699a0496cb24837daff198152, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/c75146c90fc64d3f9a554fa7021b1c59] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=38.8 K 2024-11-28T03:42:09,365 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3184713b908843d29aa1ea177058d620, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732765326596 2024-11-28T03:42:09,365 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f5b85a788cd4c2696ebc5450725f9d2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732765326596 2024-11-28T03:42:09,365 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1226dd853da04f018360022c2601d633, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732765327822 2024-11-28T03:42:09,365 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cc1613699a0496cb24837daff198152, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732765327822 2024-11-28T03:42:09,366 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ec9ce3cfb0634abfb848bef103e2cdba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765327876 2024-11-28T03:42:09,366 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c75146c90fc64d3f9a554fa7021b1c59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765327876 2024-11-28T03:42:09,374 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:09,374 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6be8c87b97e148ac91afaf8d04d2e29b is 50, key is test_row_0/A:col10/1732765327883/Put/seqid=0 2024-11-28T03:42:09,382 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:09,382 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/041f2c99e80345458caa0bcc82db3a45 is 50, key is test_row_0/B:col10/1732765327883/Put/seqid=0 2024-11-28T03:42:09,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742130_1306 (size=12915) 2024-11-28T03:42:09,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742131_1307 (size=12915) 2024-11-28T03:42:09,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T03:42:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:09,423 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T03:42:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:09,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:09,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:09,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/4e61fbf0161e4b32969b0a911b2d168a is 50, key is test_row_0/A:col10/1732765328563/Put/seqid=0 2024-11-28T03:42:09,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742132_1308 (size=12301) 2024-11-28T03:42:09,435 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/4e61fbf0161e4b32969b0a911b2d168a 2024-11-28T03:42:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/0a3eecb886e844b381f19465ed00cfb4 is 50, key is test_row_0/B:col10/1732765328563/Put/seqid=0 2024-11-28T03:42:09,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742133_1309 (size=12301) 2024-11-28T03:42:09,448 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/0a3eecb886e844b381f19465ed00cfb4 2024-11-28T03:42:09,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/8117a747d3f345f8b770a7c641c3b023 is 50, key is test_row_0/C:col10/1732765328563/Put/seqid=0 2024-11-28T03:42:09,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742134_1310 (size=12301) 2024-11-28T03:42:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:09,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765389758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:09,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765389758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765389759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:09,792 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6be8c87b97e148ac91afaf8d04d2e29b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6be8c87b97e148ac91afaf8d04d2e29b 2024-11-28T03:42:09,798 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into 6be8c87b97e148ac91afaf8d04d2e29b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:09,798 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:09,798 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=13, startTime=1732765329363; duration=0sec 2024-11-28T03:42:09,799 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:09,799 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:09,799 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:09,799 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:09,799 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:09,800 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:09,800 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/c95b538f65984809918e52a5ac6955ba, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/44abd3f9652c4f32990b14c30199d5e0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b9d81f0b1b1c4851a118a9111d76635b] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.4 K 2024-11-28T03:42:09,800 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c95b538f65984809918e52a5ac6955ba, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732765326596 2024-11-28T03:42:09,801 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44abd3f9652c4f32990b14c30199d5e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732765327822 2024-11-28T03:42:09,801 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9d81f0b1b1c4851a118a9111d76635b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765327876 2024-11-28T03:42:09,804 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/041f2c99e80345458caa0bcc82db3a45 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/041f2c99e80345458caa0bcc82db3a45 2024-11-28T03:42:09,812 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into 041f2c99e80345458caa0bcc82db3a45(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:09,812 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:09,812 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=13, startTime=1732765329363; duration=0sec 2024-11-28T03:42:09,812 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:09,812 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:09,818 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#266 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:09,819 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/18b453cf7faa483e9b320c76befdea8a is 50, key is test_row_0/C:col10/1732765327883/Put/seqid=0 2024-11-28T03:42:09,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742135_1311 (size=12915) 2024-11-28T03:42:09,833 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/18b453cf7faa483e9b320c76befdea8a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/18b453cf7faa483e9b320c76befdea8a 2024-11-28T03:42:09,837 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into 18b453cf7faa483e9b320c76befdea8a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:09,837 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:09,837 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=13, startTime=1732765329363; duration=0sec 2024-11-28T03:42:09,837 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:09,837 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:09,861 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/8117a747d3f345f8b770a7c641c3b023 2024-11-28T03:42:09,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/4e61fbf0161e4b32969b0a911b2d168a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4e61fbf0161e4b32969b0a911b2d168a 2024-11-28T03:42:09,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:09,870 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4e61fbf0161e4b32969b0a911b2d168a, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T03:42:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765389866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:09,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/0a3eecb886e844b381f19465ed00cfb4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/0a3eecb886e844b381f19465ed00cfb4 2024-11-28T03:42:09,876 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/0a3eecb886e844b381f19465ed00cfb4, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T03:42:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T03:42:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/8117a747d3f345f8b770a7c641c3b023 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8117a747d3f345f8b770a7c641c3b023 2024-11-28T03:42:09,881 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8117a747d3f345f8b770a7c641c3b023, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T03:42:09,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765389877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:09,884 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 1a2c0de76d7e5c66916b7f6387695a75 in 461ms, sequenceid=327, compaction requested=false 2024-11-28T03:42:09,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:09,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:09,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-28T03:42:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-28T03:42:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:09,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T03:42:09,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:09,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-28T03:42:09,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1550 sec 2024-11-28T03:42:09,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1630 sec 2024-11-28T03:42:09,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/223677918d044d13bf7409b5ab61c5d1 is 50, key is test_row_0/A:col10/1732765329718/Put/seqid=0 2024-11-28T03:42:09,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742136_1312 (size=12301) 2024-11-28T03:42:09,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/223677918d044d13bf7409b5ab61c5d1 2024-11-28T03:42:09,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/981034beccbd4bfcba8efc3e226128ef is 50, key is test_row_0/B:col10/1732765329718/Put/seqid=0 2024-11-28T03:42:09,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742137_1313 (size=12301) 2024-11-28T03:42:09,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765329968, max timestamp=1732765329990Num uniques 859; 2024-11-28T03:42:09,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765329968, max timestamp=1732765329990Num uniques 859; 2024-11-28T03:42:09,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:09,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765329968, max timestamp=1732765329990Num uniques 859; 2024-11-28T03:42:09,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:09,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:10,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765390077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765390082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765390086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765390196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/981034beccbd4bfcba8efc3e226128ef 2024-11-28T03:42:10,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/dbbb384329cc4dba9cc0b39a221e9b83 is 50, key is test_row_0/C:col10/1732765329718/Put/seqid=0 2024-11-28T03:42:10,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742138_1314 (size=12301) 2024-11-28T03:42:10,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/dbbb384329cc4dba9cc0b39a221e9b83 2024-11-28T03:42:10,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/223677918d044d13bf7409b5ab61c5d1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/223677918d044d13bf7409b5ab61c5d1 2024-11-28T03:42:10,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/223677918d044d13bf7409b5ab61c5d1, entries=150, sequenceid=341, filesize=12.0 K 2024-11-28T03:42:10,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/981034beccbd4bfcba8efc3e226128ef as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/981034beccbd4bfcba8efc3e226128ef 2024-11-28T03:42:10,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/981034beccbd4bfcba8efc3e226128ef, entries=150, sequenceid=341, filesize=12.0 K 2024-11-28T03:42:10,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/dbbb384329cc4dba9cc0b39a221e9b83 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/dbbb384329cc4dba9cc0b39a221e9b83 2024-11-28T03:42:10,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/dbbb384329cc4dba9cc0b39a221e9b83, entries=150, sequenceid=341, filesize=12.0 K 2024-11-28T03:42:10,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1a2c0de76d7e5c66916b7f6387695a75 in 488ms, sequenceid=341, compaction requested=true 2024-11-28T03:42:10,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:10,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:10,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:10,374 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:10,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:10,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:10,374 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:10,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:10,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:10,375 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:10,375 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:10,375 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:10,375 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6be8c87b97e148ac91afaf8d04d2e29b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4e61fbf0161e4b32969b0a911b2d168a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/223677918d044d13bf7409b5ab61c5d1] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.6 K 2024-11-28T03:42:10,375 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6be8c87b97e148ac91afaf8d04d2e29b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765327876 2024-11-28T03:42:10,376 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:10,376 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:10,376 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:10,376 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/041f2c99e80345458caa0bcc82db3a45, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/0a3eecb886e844b381f19465ed00cfb4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/981034beccbd4bfcba8efc3e226128ef] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.6 K 2024-11-28T03:42:10,376 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e61fbf0161e4b32969b0a911b2d168a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732765328544 2024-11-28T03:42:10,376 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 041f2c99e80345458caa0bcc82db3a45, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765327876 2024-11-28T03:42:10,377 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 223677918d044d13bf7409b5ab61c5d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732765329718 2024-11-28T03:42:10,377 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a3eecb886e844b381f19465ed00cfb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732765328544 2024-11-28T03:42:10,377 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 981034beccbd4bfcba8efc3e226128ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732765329718 2024-11-28T03:42:10,390 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:10,391 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/f6973eb2f5e34f368d36b7a916da9a33 is 50, key is test_row_0/A:col10/1732765329718/Put/seqid=0 2024-11-28T03:42:10,398 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:10,398 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/27d175931eb44fcbb2c9a8d6ef4c5cff is 50, key is test_row_0/B:col10/1732765329718/Put/seqid=0 2024-11-28T03:42:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:10,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T03:42:10,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:10,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:10,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:10,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:10,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:10,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:10,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/4579b60c08724b3aa82a4f54d00d6aad is 50, key is test_row_0/A:col10/1732765330400/Put/seqid=0 2024-11-28T03:42:10,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765390414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765390416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765390419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742139_1315 (size=13017) 2024-11-28T03:42:10,434 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/f6973eb2f5e34f368d36b7a916da9a33 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/f6973eb2f5e34f368d36b7a916da9a33 2024-11-28T03:42:10,441 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into f6973eb2f5e34f368d36b7a916da9a33(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:10,441 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:10,441 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=13, startTime=1732765330373; duration=0sec 2024-11-28T03:42:10,441 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:10,441 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:10,442 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:10,443 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:10,443 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:10,443 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:10,443 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/18b453cf7faa483e9b320c76befdea8a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8117a747d3f345f8b770a7c641c3b023, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/dbbb384329cc4dba9cc0b39a221e9b83] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.6 K 2024-11-28T03:42:10,444 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18b453cf7faa483e9b320c76befdea8a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765327876 2024-11-28T03:42:10,444 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8117a747d3f345f8b770a7c641c3b023, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732765328544 2024-11-28T03:42:10,444 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbbb384329cc4dba9cc0b39a221e9b83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732765329718 2024-11-28T03:42:10,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742141_1317 (size=14741) 2024-11-28T03:42:10,456 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:10,456 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/151fb73eef464b4d9ef326ace27bfb77 is 50, key is test_row_0/C:col10/1732765329718/Put/seqid=0 2024-11-28T03:42:10,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742140_1316 (size=13017) 2024-11-28T03:42:10,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/4579b60c08724b3aa82a4f54d00d6aad 2024-11-28T03:42:10,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742142_1318 (size=13017) 2024-11-28T03:42:10,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/a134157017dc4b3f9f7cfeb23114bdc1 is 50, key is test_row_0/B:col10/1732765330400/Put/seqid=0 2024-11-28T03:42:10,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742143_1319 (size=12301) 2024-11-28T03:42:10,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/a134157017dc4b3f9f7cfeb23114bdc1 2024-11-28T03:42:10,480 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/151fb73eef464b4d9ef326ace27bfb77 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/151fb73eef464b4d9ef326ace27bfb77 2024-11-28T03:42:10,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/bb8a48593b1046e6800266d80636c834 is 50, key is test_row_0/C:col10/1732765330400/Put/seqid=0 2024-11-28T03:42:10,485 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into 151fb73eef464b4d9ef326ace27bfb77(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:10,485 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:10,485 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=13, startTime=1732765330374; duration=0sec 2024-11-28T03:42:10,485 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:10,485 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:10,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742144_1320 (size=12301) 2024-11-28T03:42:10,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765390526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765390527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765390527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765390737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765390737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:10,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765390737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:10,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:10,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:10,864 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/27d175931eb44fcbb2c9a8d6ef4c5cff as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/27d175931eb44fcbb2c9a8d6ef4c5cff 2024-11-28T03:42:10,879 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into 27d175931eb44fcbb2c9a8d6ef4c5cff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:10,879 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:10,880 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=13, startTime=1732765330374; duration=0sec 2024-11-28T03:42:10,880 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:10,880 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:10,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/bb8a48593b1046e6800266d80636c834 2024-11-28T03:42:10,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/4579b60c08724b3aa82a4f54d00d6aad as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4579b60c08724b3aa82a4f54d00d6aad 2024-11-28T03:42:10,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4579b60c08724b3aa82a4f54d00d6aad, entries=200, sequenceid=378, filesize=14.4 K 2024-11-28T03:42:10,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/a134157017dc4b3f9f7cfeb23114bdc1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a134157017dc4b3f9f7cfeb23114bdc1 2024-11-28T03:42:10,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a134157017dc4b3f9f7cfeb23114bdc1, entries=150, sequenceid=378, filesize=12.0 K 2024-11-28T03:42:10,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/bb8a48593b1046e6800266d80636c834 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/bb8a48593b1046e6800266d80636c834 2024-11-28T03:42:10,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/bb8a48593b1046e6800266d80636c834, entries=150, sequenceid=378, filesize=12.0 K 2024-11-28T03:42:10,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 1a2c0de76d7e5c66916b7f6387695a75 in 517ms, sequenceid=378, compaction requested=false 2024-11-28T03:42:10,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:11,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T03:42:11,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:11,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:11,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:11,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:11,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:11,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:11,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/41c2b61660d241ae9cd624a057b3312e is 50, key is test_row_0/A:col10/1732765331048/Put/seqid=0 2024-11-28T03:42:11,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742145_1321 (size=12301) 2024-11-28T03:42:11,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/41c2b61660d241ae9cd624a057b3312e 2024-11-28T03:42:11,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/2f41de79b0804763bc21acfe7ae3f7a7 is 50, key is test_row_0/B:col10/1732765331048/Put/seqid=0 2024-11-28T03:42:11,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765391093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:11,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765391099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:11,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765391103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:11,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742146_1322 (size=12301) 2024-11-28T03:42:11,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/2f41de79b0804763bc21acfe7ae3f7a7 2024-11-28T03:42:11,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/e7f92a7f02354dafbb28be97fb90c799 is 50, key is test_row_0/C:col10/1732765331048/Put/seqid=0 2024-11-28T03:42:11,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742147_1323 (size=12301) 2024-11-28T03:42:11,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/e7f92a7f02354dafbb28be97fb90c799 2024-11-28T03:42:11,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/41c2b61660d241ae9cd624a057b3312e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/41c2b61660d241ae9cd624a057b3312e 2024-11-28T03:42:11,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/41c2b61660d241ae9cd624a057b3312e, entries=150, sequenceid=393, filesize=12.0 K 2024-11-28T03:42:11,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/2f41de79b0804763bc21acfe7ae3f7a7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/2f41de79b0804763bc21acfe7ae3f7a7 2024-11-28T03:42:11,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/2f41de79b0804763bc21acfe7ae3f7a7, entries=150, sequenceid=393, filesize=12.0 K 2024-11-28T03:42:11,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/e7f92a7f02354dafbb28be97fb90c799 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/e7f92a7f02354dafbb28be97fb90c799 2024-11-28T03:42:11,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/e7f92a7f02354dafbb28be97fb90c799, entries=150, sequenceid=393, filesize=12.0 K 2024-11-28T03:42:11,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1a2c0de76d7e5c66916b7f6387695a75 in 142ms, sequenceid=393, compaction requested=true 2024-11-28T03:42:11,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:11,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:11,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:11,193 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:11,193 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:11,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:11,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:11,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:11,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:11,195 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:11,195 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:11,195 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:11,195 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/f6973eb2f5e34f368d36b7a916da9a33, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4579b60c08724b3aa82a4f54d00d6aad, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/41c2b61660d241ae9cd624a057b3312e] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=39.1 K 2024-11-28T03:42:11,196 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:11,196 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:11,196 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:11,196 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/27d175931eb44fcbb2c9a8d6ef4c5cff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a134157017dc4b3f9f7cfeb23114bdc1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/2f41de79b0804763bc21acfe7ae3f7a7] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.7 K 2024-11-28T03:42:11,197 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6973eb2f5e34f368d36b7a916da9a33, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732765329718 2024-11-28T03:42:11,197 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 27d175931eb44fcbb2c9a8d6ef4c5cff, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732765329718 2024-11-28T03:42:11,197 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting a134157017dc4b3f9f7cfeb23114bdc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732765330065 2024-11-28T03:42:11,197 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f41de79b0804763bc21acfe7ae3f7a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732765330417 2024-11-28T03:42:11,198 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4579b60c08724b3aa82a4f54d00d6aad, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732765330058 2024-11-28T03:42:11,199 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c2b61660d241ae9cd624a057b3312e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732765330417 2024-11-28T03:42:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:11,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765331090, max timestamp=1732765331213Num uniques 859; 2024-11-28T03:42:11,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:11,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765331090, max timestamp=1732765331213Num uniques 859; 2024-11-28T03:42:11,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:11,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-28T03:42:11,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765331090, max timestamp=1732765331213Num uniques 859; 2024-11-28T03:42:11,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:11,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:11,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:11,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:11,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:11,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:11,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:11,230 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:11,231 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/f4f506ce0d894074b0b81313db2053f7 is 50, key is test_row_0/B:col10/1732765331048/Put/seqid=0 2024-11-28T03:42:11,250 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:11,251 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/1342a1867dfe4caf89934596d01afa56 is 50, key is test_row_0/A:col10/1732765331048/Put/seqid=0 2024-11-28T03:42:11,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/cd135b8c46b64d0091ca5470792c15db is 50, key is test_row_0/A:col10/1732765331091/Put/seqid=0 2024-11-28T03:42:11,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765331257, max timestamp=1732765331272Num uniques 859; 2024-11-28T03:42:11,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765331257, max timestamp=1732765331272Num uniques 859; 2024-11-28T03:42:11,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:11,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:11,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765331257, max timestamp=1732765331272Num uniques 859; 2024-11-28T03:42:11,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:11,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742148_1324 (size=13119) 2024-11-28T03:42:11,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742149_1325 (size=13119) 2024-11-28T03:42:11,288 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/f4f506ce0d894074b0b81313db2053f7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f4f506ce0d894074b0b81313db2053f7 2024-11-28T03:42:11,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742150_1326 (size=14741) 2024-11-28T03:42:11,298 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into f4f506ce0d894074b0b81313db2053f7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:11,298 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:11,298 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=13, startTime=1732765331193; duration=0sec 2024-11-28T03:42:11,299 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:11,299 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:11,299 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:11,301 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:11,301 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:11,301 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:11,302 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/151fb73eef464b4d9ef326ace27bfb77, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/bb8a48593b1046e6800266d80636c834, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/e7f92a7f02354dafbb28be97fb90c799] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.7 K 2024-11-28T03:42:11,302 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 151fb73eef464b4d9ef326ace27bfb77, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732765329718 2024-11-28T03:42:11,302 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting bb8a48593b1046e6800266d80636c834, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732765330065 2024-11-28T03:42:11,302 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e7f92a7f02354dafbb28be97fb90c799, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732765330417 2024-11-28T03:42:11,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765331311, max timestamp=1732765331321Num uniques 1018; 2024-11-28T03:42:11,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T03:42:11,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765331311, max timestamp=1732765331321Num uniques 1018; 2024-11-28T03:42:11,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T03:42:11,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765331311, max timestamp=1732765331321Num uniques 1018; 2024-11-28T03:42:11,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T03:42:11,342 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:11,343 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/a970bc56c04d4d3a99aea4f457dc8c8a is 50, key is test_row_0/C:col10/1732765331048/Put/seqid=0 2024-11-28T03:42:11,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742151_1327 (size=13119) 2024-11-28T03:42:11,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732765331367, max timestamp=1732765331380Num uniques 1027; 2024-11-28T03:42:11,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T03:42:11,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732765331367, max timestamp=1732765331380Num uniques 1027; 2024-11-28T03:42:11,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T03:42:11,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732765331367, max timestamp=1732765331380Num uniques 1027; 2024-11-28T03:42:11,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T03:42:11,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732765331410, max timestamp=1732765331422Num uniques 1036; 2024-11-28T03:42:11,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-28T03:42:11,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732765331410, max timestamp=1732765331422Num uniques 1036; 2024-11-28T03:42:11,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-28T03:42:11,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732765331410, max timestamp=1732765331422Num uniques 1036; 2024-11-28T03:42:11,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-28T03:42:11,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732765331448, max timestamp=1732765331465Num uniques 1045; 2024-11-28T03:42:11,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732765331448, max timestamp=1732765331465Num uniques 1045; 2024-11-28T03:42:11,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-28T03:42:11,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-28T03:42:11,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732765331448, max timestamp=1732765331465Num uniques 1045; 2024-11-28T03:42:11,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-28T03:42:11,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732765331495, max timestamp=1732765331505Num uniques 1104; 2024-11-28T03:42:11,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-28T03:42:11,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732765331495, max timestamp=1732765331505Num uniques 1104; 2024-11-28T03:42:11,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-28T03:42:11,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732765331495, max timestamp=1732765331505Num uniques 1104; 2024-11-28T03:42:11,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-28T03:42:11,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732765331534, max timestamp=1732765331545Num uniques 1113; 2024-11-28T03:42:11,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732765331534, max timestamp=1732765331545Num uniques 1113; 2024-11-28T03:42:11,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-28T03:42:11,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-28T03:42:11,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732765331534, max timestamp=1732765331545Num uniques 1113; 2024-11-28T03:42:11,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-28T03:42:11,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732765331571, max timestamp=1732765331585Num uniques 1072; 2024-11-28T03:42:11,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-28T03:42:11,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732765331571, max timestamp=1732765331585Num uniques 1072; 2024-11-28T03:42:11,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-28T03:42:11,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732765331571, max timestamp=1732765331585Num uniques 1072; 2024-11-28T03:42:11,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-28T03:42:11,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732765331602, max timestamp=1732765331644Num uniques 1131; 2024-11-28T03:42:11,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732765331602, max timestamp=1732765331644Num uniques 1131; 2024-11-28T03:42:11,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-28T03:42:11,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-28T03:42:11,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732765331602, max timestamp=1732765331644Num uniques 1131; 2024-11-28T03:42:11,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-28T03:42:11,689 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/1342a1867dfe4caf89934596d01afa56 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1342a1867dfe4caf89934596d01afa56 2024-11-28T03:42:11,696 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into 1342a1867dfe4caf89934596d01afa56(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:11,696 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:11,696 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=13, startTime=1732765331193; duration=0sec 2024-11-28T03:42:11,696 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:11,696 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:11,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/cd135b8c46b64d0091ca5470792c15db 2024-11-28T03:42:11,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1e62516a7f334e95aa09391b31b497ed is 50, key is test_row_0/B:col10/1732765331091/Put/seqid=0 2024-11-28T03:42:11,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732765331697, max timestamp=1732765331709Num uniques 1090; 2024-11-28T03:42:11,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-28T03:42:11,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732765331697, max timestamp=1732765331709Num uniques 1090; 2024-11-28T03:42:11,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-28T03:42:11,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732765331697, max timestamp=1732765331709Num uniques 1090; 2024-11-28T03:42:11,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-28T03:42:11,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742152_1328 (size=12301) 2024-11-28T03:42:11,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1e62516a7f334e95aa09391b31b497ed 2024-11-28T03:42:11,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/eec441ecd20e46dfb1f2bdece26a96d8 is 50, key is test_row_0/C:col10/1732765331091/Put/seqid=0 2024-11-28T03:42:11,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742153_1329 (size=12301) 2024-11-28T03:42:11,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/eec441ecd20e46dfb1f2bdece26a96d8 2024-11-28T03:42:11,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/cd135b8c46b64d0091ca5470792c15db as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/cd135b8c46b64d0091ca5470792c15db 2024-11-28T03:42:11,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/cd135b8c46b64d0091ca5470792c15db, entries=200, sequenceid=414, filesize=14.4 K 2024-11-28T03:42:11,772 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/a970bc56c04d4d3a99aea4f457dc8c8a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/a970bc56c04d4d3a99aea4f457dc8c8a 2024-11-28T03:42:11,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1e62516a7f334e95aa09391b31b497ed as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1e62516a7f334e95aa09391b31b497ed 2024-11-28T03:42:11,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9106, totalHeapSize=20936, min timestamp=1732765331748, max timestamp=1732765331772Num uniques 1049; 2024-11-28T03:42:11,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1732765331748, max timestamp=1732765331772Num uniques 1049; 2024-11-28T03:42:11,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48036, new segment data size=9106, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-28T03:42:11,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48034, new segment data size=9104, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-28T03:42:11,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9112, totalHeapSize=20936, min timestamp=1732765331748, max timestamp=1732765331772Num uniques 1049; 2024-11-28T03:42:11,781 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into a970bc56c04d4d3a99aea4f457dc8c8a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:11,781 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:11,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48042, new segment data size=9112, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-28T03:42:11,781 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=13, startTime=1732765331193; duration=0sec 2024-11-28T03:42:11,781 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:11,782 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:11,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1e62516a7f334e95aa09391b31b497ed, entries=150, sequenceid=414, filesize=12.0 K 2024-11-28T03:42:11,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/eec441ecd20e46dfb1f2bdece26a96d8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/eec441ecd20e46dfb1f2bdece26a96d8 2024-11-28T03:42:11,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/eec441ecd20e46dfb1f2bdece26a96d8, entries=150, sequenceid=414, filesize=12.0 K 2024-11-28T03:42:11,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=60.38 KB/61830 for 1a2c0de76d7e5c66916b7f6387695a75 in 575ms, sequenceid=414, compaction requested=false 2024-11-28T03:42:11,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:11,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=67.09 KB heapSize=182.02 KB 2024-11-28T03:42:11,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:11,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:11,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:11,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:11,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:11,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:11,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/d90b0e90831945d9b9a4f4a7ac7771cd is 50, key is test_row_0/A:col10/1732765331799/Put/seqid=0 2024-11-28T03:42:11,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742154_1330 (size=12301) 2024-11-28T03:42:11,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/d90b0e90831945d9b9a4f4a7ac7771cd 2024-11-28T03:42:11,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765391851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:11,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765391852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:11,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765391852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:11,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:11,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:11,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/70c1636d5cff4ec5b9c06380575d75a7 is 50, key is test_row_0/B:col10/1732765331799/Put/seqid=0 2024-11-28T03:42:11,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742155_1331 (size=12301) 2024-11-28T03:42:11,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/70c1636d5cff4ec5b9c06380575d75a7 2024-11-28T03:42:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T03:42:11,897 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:11,897 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-28T03:42:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T03:42:11,900 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:11,900 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:11,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:11,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/abaa4999e9ee43108657dcdbe4554bf9 is 50, key is test_row_0/C:col10/1732765331799/Put/seqid=0 2024-11-28T03:42:11,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742156_1332 (size=12301) 2024-11-28T03:42:11,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765391966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:11,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:11,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765391966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:11,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:11,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:11,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765391966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:11,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:11,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:11,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T03:42:12,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T03:42:12,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:12,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:12,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:12,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:12,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:12,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765392176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765392176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765392176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T03:42:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:12,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T03:42:12,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/abaa4999e9ee43108657dcdbe4554bf9 2024-11-28T03:42:12,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/d90b0e90831945d9b9a4f4a7ac7771cd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/d90b0e90831945d9b9a4f4a7ac7771cd 2024-11-28T03:42:12,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/d90b0e90831945d9b9a4f4a7ac7771cd, entries=150, sequenceid=617, filesize=12.0 K 2024-11-28T03:42:12,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/70c1636d5cff4ec5b9c06380575d75a7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/70c1636d5cff4ec5b9c06380575d75a7 2024-11-28T03:42:12,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/70c1636d5cff4ec5b9c06380575d75a7, entries=150, sequenceid=617, filesize=12.0 K 2024-11-28T03:42:12,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/abaa4999e9ee43108657dcdbe4554bf9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/abaa4999e9ee43108657dcdbe4554bf9 2024-11-28T03:42:12,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/abaa4999e9ee43108657dcdbe4554bf9, entries=150, sequenceid=617, filesize=12.0 K 2024-11-28T03:42:12,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~221.81 KB/227136, currentSize=107.34 KB/109920 for 1a2c0de76d7e5c66916b7f6387695a75 in 541ms, sequenceid=617, compaction requested=true 2024-11-28T03:42:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:12,341 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:12,341 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:12,342 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:12,342 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:12,342 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:12,342 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:12,343 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:12,343 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:12,343 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1342a1867dfe4caf89934596d01afa56, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/cd135b8c46b64d0091ca5470792c15db, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/d90b0e90831945d9b9a4f4a7ac7771cd] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=39.2 K 2024-11-28T03:42:12,343 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f4f506ce0d894074b0b81313db2053f7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1e62516a7f334e95aa09391b31b497ed, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/70c1636d5cff4ec5b9c06380575d75a7] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.8 K 2024-11-28T03:42:12,343 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f506ce0d894074b0b81313db2053f7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732765330417 2024-11-28T03:42:12,343 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1342a1867dfe4caf89934596d01afa56, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732765330417 2024-11-28T03:42:12,344 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting cd135b8c46b64d0091ca5470792c15db, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732765331090 2024-11-28T03:42:12,344 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e62516a7f334e95aa09391b31b497ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732765331091 2024-11-28T03:42:12,344 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c1636d5cff4ec5b9c06380575d75a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732765331797 2024-11-28T03:42:12,344 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d90b0e90831945d9b9a4f4a7ac7771cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732765331797 2024-11-28T03:42:12,352 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:12,352 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:12,352 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/8e894f3d0e294f969eb123a6989c563e is 50, key is test_row_0/B:col10/1732765331799/Put/seqid=0 2024-11-28T03:42:12,353 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/72cdb121f04743e7ac4b85de9d9650a2 is 50, key is test_row_0/A:col10/1732765331799/Put/seqid=0 2024-11-28T03:42:12,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T03:42:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:12,360 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:42:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:12,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/dbb29544708845f2b570b0ed2cf71551 is 50, key is test_row_0/A:col10/1732765331849/Put/seqid=0 2024-11-28T03:42:12,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742157_1333 (size=13221) 2024-11-28T03:42:12,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742158_1334 (size=13221) 2024-11-28T03:42:12,430 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/72cdb121f04743e7ac4b85de9d9650a2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/72cdb121f04743e7ac4b85de9d9650a2 2024-11-28T03:42:12,437 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into 72cdb121f04743e7ac4b85de9d9650a2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:12,437 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:12,437 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=13, startTime=1732765332341; duration=0sec 2024-11-28T03:42:12,437 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:12,437 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:12,437 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:12,438 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:12,439 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:12,439 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:12,439 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/a970bc56c04d4d3a99aea4f457dc8c8a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/eec441ecd20e46dfb1f2bdece26a96d8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/abaa4999e9ee43108657dcdbe4554bf9] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.8 K 2024-11-28T03:42:12,439 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting a970bc56c04d4d3a99aea4f457dc8c8a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732765330417 2024-11-28T03:42:12,440 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting eec441ecd20e46dfb1f2bdece26a96d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732765331091 2024-11-28T03:42:12,440 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting abaa4999e9ee43108657dcdbe4554bf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732765331797 2024-11-28T03:42:12,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742159_1335 (size=12301) 2024-11-28T03:42:12,445 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/dbb29544708845f2b570b0ed2cf71551 2024-11-28T03:42:12,451 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:12,452 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/3971718be5fe4d46808cb79d00901c7f is 50, key is test_row_0/C:col10/1732765331799/Put/seqid=0 2024-11-28T03:42:12,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/fbb9352a2b0745a78667e1bb0b968092 is 50, key is test_row_0/B:col10/1732765331849/Put/seqid=0 2024-11-28T03:42:12,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742161_1337 (size=12301) 2024-11-28T03:42:12,479 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/fbb9352a2b0745a78667e1bb0b968092 2024-11-28T03:42:12,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742160_1336 (size=13221) 2024-11-28T03:42:12,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/206abc76834349d080666bad8c6bc7c5 is 50, key is test_row_0/C:col10/1732765331849/Put/seqid=0 2024-11-28T03:42:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:12,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:12,499 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/3971718be5fe4d46808cb79d00901c7f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3971718be5fe4d46808cb79d00901c7f 2024-11-28T03:42:12,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742162_1338 (size=12301) 2024-11-28T03:42:12,501 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/206abc76834349d080666bad8c6bc7c5 2024-11-28T03:42:12,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/dbb29544708845f2b570b0ed2cf71551 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/dbb29544708845f2b570b0ed2cf71551 2024-11-28T03:42:12,507 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into 3971718be5fe4d46808cb79d00901c7f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:12,507 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:12,507 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=13, startTime=1732765332341; duration=0sec 2024-11-28T03:42:12,508 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:12,508 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:12,510 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/dbb29544708845f2b570b0ed2cf71551, entries=150, sequenceid=636, filesize=12.0 K 2024-11-28T03:42:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/fbb9352a2b0745a78667e1bb0b968092 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fbb9352a2b0745a78667e1bb0b968092 2024-11-28T03:42:12,515 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fbb9352a2b0745a78667e1bb0b968092, entries=150, sequenceid=636, filesize=12.0 K 2024-11-28T03:42:12,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/206abc76834349d080666bad8c6bc7c5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/206abc76834349d080666bad8c6bc7c5 2024-11-28T03:42:12,520 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/206abc76834349d080666bad8c6bc7c5, entries=150, sequenceid=636, filesize=12.0 K 2024-11-28T03:42:12,521 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1a2c0de76d7e5c66916b7f6387695a75 in 161ms, sequenceid=636, compaction requested=false 2024-11-28T03:42:12,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:12,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:12,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-28T03:42:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-28T03:42:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:12,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:42:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:12,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-28T03:42:12,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 621 msec 2024-11-28T03:42:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T03:42:12,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 629 msec 2024-11-28T03:42:12,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/4aec824ed01b45eb9da9c2eb541467c9 is 50, key is test_row_0/A:col10/1732765332516/Put/seqid=0 2024-11-28T03:42:12,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742163_1339 (size=14741) 2024-11-28T03:42:12,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/4aec824ed01b45eb9da9c2eb541467c9 2024-11-28T03:42:12,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/759f16691344423394f7ea1247232135 is 50, key is test_row_0/B:col10/1732765332516/Put/seqid=0 2024-11-28T03:42:12,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742164_1340 (size=12301) 2024-11-28T03:42:12,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765392558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765392559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765392559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765392666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765392677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765392677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,820 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/8e894f3d0e294f969eb123a6989c563e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8e894f3d0e294f969eb123a6989c563e 2024-11-28T03:42:12,826 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into 8e894f3d0e294f969eb123a6989c563e(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:12,826 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:12,826 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=13, startTime=1732765332341; duration=0sec 2024-11-28T03:42:12,827 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:12,827 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:12,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765392876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765392887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765392887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:12,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/759f16691344423394f7ea1247232135 2024-11-28T03:42:12,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/b8e4da13084a4926848ba96ad5c97d27 is 50, key is test_row_0/C:col10/1732765332516/Put/seqid=0 2024-11-28T03:42:12,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742165_1341 (size=12301) 2024-11-28T03:42:13,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38398 deadline: 1732765393017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,020 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_0' in 1a2c0de76d7e5c66916b7f6387695a75 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:13,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38424 deadline: 1732765393026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,031 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in 1a2c0de76d7e5c66916b7f6387695a75 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T03:42:13,037 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:13,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-28T03:42:13,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T03:42:13,039 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:13,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:13,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:13,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T03:42:13,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765393187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:13,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T03:42:13,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:13,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:13,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:13,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:13,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:13,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765393196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765393196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:13,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T03:42:13,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:13,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:13,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:13,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:13,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T03:42:13,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/b8e4da13084a4926848ba96ad5c97d27 2024-11-28T03:42:13,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/4aec824ed01b45eb9da9c2eb541467c9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4aec824ed01b45eb9da9c2eb541467c9 2024-11-28T03:42:13,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4aec824ed01b45eb9da9c2eb541467c9, entries=200, sequenceid=653, filesize=14.4 K 2024-11-28T03:42:13,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/759f16691344423394f7ea1247232135 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/759f16691344423394f7ea1247232135 2024-11-28T03:42:13,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/759f16691344423394f7ea1247232135, entries=150, sequenceid=653, filesize=12.0 K 2024-11-28T03:42:13,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/b8e4da13084a4926848ba96ad5c97d27 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b8e4da13084a4926848ba96ad5c97d27 2024-11-28T03:42:13,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b8e4da13084a4926848ba96ad5c97d27, entries=150, sequenceid=653, filesize=12.0 K 2024-11-28T03:42:13,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 1a2c0de76d7e5c66916b7f6387695a75 in 878ms, sequenceid=653, compaction requested=true 2024-11-28T03:42:13,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:13,403 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:13,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:13,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:13,403 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:13,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:13,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:13,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:13,405 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:13,405 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:13,405 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:13,405 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/72cdb121f04743e7ac4b85de9d9650a2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/dbb29544708845f2b570b0ed2cf71551, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4aec824ed01b45eb9da9c2eb541467c9] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=39.3 K 2024-11-28T03:42:13,405 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:13,405 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:13,405 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:13,406 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8e894f3d0e294f969eb123a6989c563e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fbb9352a2b0745a78667e1bb0b968092, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/759f16691344423394f7ea1247232135] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.9 K 2024-11-28T03:42:13,406 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72cdb121f04743e7ac4b85de9d9650a2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732765331797 2024-11-28T03:42:13,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:13,406 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e894f3d0e294f969eb123a6989c563e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732765331797 2024-11-28T03:42:13,406 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbb29544708845f2b570b0ed2cf71551, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1732765331838 2024-11-28T03:42:13,406 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb9352a2b0745a78667e1bb0b968092, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1732765331838 2024-11-28T03:42:13,407 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 759f16691344423394f7ea1247232135, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1732765332516 2024-11-28T03:42:13,407 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aec824ed01b45eb9da9c2eb541467c9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1732765332516 2024-11-28T03:42:13,416 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:13,417 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/9136f7865f1d472580cc78b7a4cd5b02 is 50, key is test_row_0/B:col10/1732765332516/Put/seqid=0 2024-11-28T03:42:13,419 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:13,419 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/71c4b655cdad4df7b2c239c14900fb77 is 50, key is test_row_0/A:col10/1732765332516/Put/seqid=0 2024-11-28T03:42:13,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742166_1342 (size=13323) 2024-11-28T03:42:13,454 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/9136f7865f1d472580cc78b7a4cd5b02 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/9136f7865f1d472580cc78b7a4cd5b02 2024-11-28T03:42:13,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742167_1343 (size=13323) 2024-11-28T03:42:13,469 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into 9136f7865f1d472580cc78b7a4cd5b02(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:13,469 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:13,471 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=13, startTime=1732765333403; duration=0sec 2024-11-28T03:42:13,471 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:13,473 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:13,473 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:13,480 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:13,480 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:13,481 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:13,481 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3971718be5fe4d46808cb79d00901c7f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/206abc76834349d080666bad8c6bc7c5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b8e4da13084a4926848ba96ad5c97d27] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=36.9 K 2024-11-28T03:42:13,482 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3971718be5fe4d46808cb79d00901c7f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732765331797 2024-11-28T03:42:13,483 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 206abc76834349d080666bad8c6bc7c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1732765331838 2024-11-28T03:42:13,484 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting b8e4da13084a4926848ba96ad5c97d27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1732765332516 2024-11-28T03:42:13,485 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/71c4b655cdad4df7b2c239c14900fb77 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/71c4b655cdad4df7b2c239c14900fb77 2024-11-28T03:42:13,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T03:42:13,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:13,500 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:42:13,502 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into 71c4b655cdad4df7b2c239c14900fb77(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:13,502 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:13,502 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=13, startTime=1732765333403; duration=0sec 2024-11-28T03:42:13,502 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:13,502 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:13,509 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:13,510 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/4573bd04c48f4f06b5d39624a5a417d6 is 50, key is test_row_0/C:col10/1732765332516/Put/seqid=0 2024-11-28T03:42:13,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/db6da8edb2284ab3a5ea6a9d87e9ab66 is 50, key is test_row_0/A:col10/1732765332550/Put/seqid=0 2024-11-28T03:42:13,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742168_1344 (size=13323) 2024-11-28T03:42:13,586 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/4573bd04c48f4f06b5d39624a5a417d6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/4573bd04c48f4f06b5d39624a5a417d6 2024-11-28T03:42:13,595 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into 4573bd04c48f4f06b5d39624a5a417d6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:13,595 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:13,595 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=13, startTime=1732765333404; duration=0sec 2024-11-28T03:42:13,595 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:13,595 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:13,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742169_1345 (size=12301) 2024-11-28T03:42:13,610 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=674 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/db6da8edb2284ab3a5ea6a9d87e9ab66 2024-11-28T03:42:13,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/839b7bd02d68414ca22affee8b7919b9 is 50, key is test_row_0/B:col10/1732765332550/Put/seqid=0 2024-11-28T03:42:13,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742170_1346 (size=12301) 2024-11-28T03:42:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T03:42:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:13,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765393727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765393728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765393730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:13,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765393836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765393836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765393836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765394046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765394046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765394046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,051 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=674 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/839b7bd02d68414ca22affee8b7919b9 2024-11-28T03:42:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/3a1717d8d35548a5be3da3aeefdaca51 is 50, key is test_row_0/C:col10/1732765332550/Put/seqid=0 2024-11-28T03:42:14,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742171_1347 (size=12301) 2024-11-28T03:42:14,077 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=674 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/3a1717d8d35548a5be3da3aeefdaca51 2024-11-28T03:42:14,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/db6da8edb2284ab3a5ea6a9d87e9ab66 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/db6da8edb2284ab3a5ea6a9d87e9ab66 2024-11-28T03:42:14,088 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/db6da8edb2284ab3a5ea6a9d87e9ab66, entries=150, sequenceid=674, filesize=12.0 K 2024-11-28T03:42:14,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/839b7bd02d68414ca22affee8b7919b9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/839b7bd02d68414ca22affee8b7919b9 2024-11-28T03:42:14,094 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/839b7bd02d68414ca22affee8b7919b9, entries=150, sequenceid=674, filesize=12.0 K 2024-11-28T03:42:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/3a1717d8d35548a5be3da3aeefdaca51 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3a1717d8d35548a5be3da3aeefdaca51 2024-11-28T03:42:14,101 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3a1717d8d35548a5be3da3aeefdaca51, entries=150, sequenceid=674, filesize=12.0 K 2024-11-28T03:42:14,102 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1a2c0de76d7e5c66916b7f6387695a75 in 602ms, sequenceid=674, compaction requested=false 2024-11-28T03:42:14,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:14,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:14,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-28T03:42:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-28T03:42:14,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-28T03:42:14,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0640 sec 2024-11-28T03:42:14,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0690 sec 2024-11-28T03:42:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T03:42:14,177 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:14,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-28T03:42:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T03:42:14,179 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:14,180 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:14,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T03:42:14,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T03:42:14,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:14,333 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:42:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:14,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/646851f40a9d498da569c65366aa15a8 is 50, key is test_row_0/A:col10/1732765333726/Put/seqid=0 2024-11-28T03:42:14,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742172_1348 (size=12301) 2024-11-28T03:42:14,344 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/646851f40a9d498da569c65366aa15a8 2024-11-28T03:42:14,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/fb461f044eba41718ab69fc81c8b3c52 is 50, key is test_row_0/B:col10/1732765333726/Put/seqid=0 2024-11-28T03:42:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:14,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:14,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742173_1349 (size=12301) 2024-11-28T03:42:14,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765394386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765394387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765394388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T03:42:14,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765394496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765394497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765394497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765394706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765394707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765394707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:14,763 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/fb461f044eba41718ab69fc81c8b3c52 2024-11-28T03:42:14,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/8c195c479c064bee972368fbc9f3512f is 50, key is test_row_0/C:col10/1732765333726/Put/seqid=0 2024-11-28T03:42:14,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742174_1350 (size=12301) 2024-11-28T03:42:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T03:42:15,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765395016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765395016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765395016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,195 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/8c195c479c064bee972368fbc9f3512f 2024-11-28T03:42:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/646851f40a9d498da569c65366aa15a8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/646851f40a9d498da569c65366aa15a8 2024-11-28T03:42:15,204 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/646851f40a9d498da569c65366aa15a8, entries=150, sequenceid=689, filesize=12.0 K 2024-11-28T03:42:15,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/fb461f044eba41718ab69fc81c8b3c52 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fb461f044eba41718ab69fc81c8b3c52 2024-11-28T03:42:15,208 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fb461f044eba41718ab69fc81c8b3c52, entries=150, sequenceid=689, filesize=12.0 K 2024-11-28T03:42:15,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/8c195c479c064bee972368fbc9f3512f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8c195c479c064bee972368fbc9f3512f 2024-11-28T03:42:15,213 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8c195c479c064bee972368fbc9f3512f, entries=150, sequenceid=689, filesize=12.0 K 2024-11-28T03:42:15,214 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1a2c0de76d7e5c66916b7f6387695a75 in 881ms, sequenceid=689, compaction requested=true 2024-11-28T03:42:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-28T03:42:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-28T03:42:15,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-28T03:42:15,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0350 sec 2024-11-28T03:42:15,219 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.0410 sec 2024-11-28T03:42:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T03:42:15,317 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:15,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-28T03:42:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T03:42:15,319 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:15,320 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:15,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T03:42:15,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T03:42:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:15,473 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:42:15,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:15,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:15,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:15,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:15,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:15,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:15,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/77d1ca1c2ed5450295c52bee45e10031 is 50, key is test_row_0/A:col10/1732765334386/Put/seqid=0 2024-11-28T03:42:15,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742175_1351 (size=12301) 2024-11-28T03:42:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:15,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765395562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765395563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765395564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T03:42:15,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765395676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765395676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765395676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,885 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=707 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/77d1ca1c2ed5450295c52bee45e10031 2024-11-28T03:42:15,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765395887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765395887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765395887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/58e0f906d8e1454bb0d0896e5ed64bcc is 50, key is test_row_0/B:col10/1732765334386/Put/seqid=0 2024-11-28T03:42:15,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742176_1352 (size=12301) 2024-11-28T03:42:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T03:42:16,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765396196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765396196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:16,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765396197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:16,302 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=707 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/58e0f906d8e1454bb0d0896e5ed64bcc 2024-11-28T03:42:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/ec85dd38c1634a65a5a825752ffa3224 is 50, key is test_row_0/C:col10/1732765334386/Put/seqid=0 2024-11-28T03:42:16,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742177_1353 (size=12301) 2024-11-28T03:42:16,317 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=707 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/ec85dd38c1634a65a5a825752ffa3224 2024-11-28T03:42:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/77d1ca1c2ed5450295c52bee45e10031 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/77d1ca1c2ed5450295c52bee45e10031 2024-11-28T03:42:16,326 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/77d1ca1c2ed5450295c52bee45e10031, entries=150, sequenceid=707, filesize=12.0 K 2024-11-28T03:42:16,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/58e0f906d8e1454bb0d0896e5ed64bcc as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58e0f906d8e1454bb0d0896e5ed64bcc 2024-11-28T03:42:16,330 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58e0f906d8e1454bb0d0896e5ed64bcc, entries=150, sequenceid=707, filesize=12.0 K 2024-11-28T03:42:16,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/ec85dd38c1634a65a5a825752ffa3224 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ec85dd38c1634a65a5a825752ffa3224 2024-11-28T03:42:16,336 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ec85dd38c1634a65a5a825752ffa3224, entries=150, sequenceid=707, filesize=12.0 K 2024-11-28T03:42:16,336 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1a2c0de76d7e5c66916b7f6387695a75 in 863ms, sequenceid=707, compaction requested=true 2024-11-28T03:42:16,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:16,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:16,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-28T03:42:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-28T03:42:16,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-28T03:42:16,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0180 sec 2024-11-28T03:42:16,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0250 sec 2024-11-28T03:42:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T03:42:16,457 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:16,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-28T03:42:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T03:42:16,460 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:16,461 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:16,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T03:42:16,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-28T03:42:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:16,617 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:42:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:16,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6475f53d0688400797cb7300e55d82ac is 50, key is test_row_0/A:col10/1732765335561/Put/seqid=0 2024-11-28T03:42:16,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742178_1354 (size=12301) 2024-11-28T03:42:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:16,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:16,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765396732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:16,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:16,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765396737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:16,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765396738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T03:42:16,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765396847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:16,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765396847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:16,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765396847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:17,039 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6475f53d0688400797cb7300e55d82ac 2024-11-28T03:42:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1681a3a0c47f42de9c690bb1db1169cc is 50, key is test_row_0/B:col10/1732765335561/Put/seqid=0 2024-11-28T03:42:17,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742179_1355 (size=12301) 2024-11-28T03:42:17,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765397056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:17,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765397057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765397057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T03:42:17,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765397366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765397366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765397366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:17,453 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1681a3a0c47f42de9c690bb1db1169cc 2024-11-28T03:42:17,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/38de02a0168049bcbd8fb13ff1f6ca98 is 50, key is test_row_0/C:col10/1732765335561/Put/seqid=0 2024-11-28T03:42:17,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742180_1356 (size=12301) 2024-11-28T03:42:17,464 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/38de02a0168049bcbd8fb13ff1f6ca98 2024-11-28T03:42:17,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6475f53d0688400797cb7300e55d82ac as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6475f53d0688400797cb7300e55d82ac 2024-11-28T03:42:17,471 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6475f53d0688400797cb7300e55d82ac, entries=150, sequenceid=721, filesize=12.0 K 2024-11-28T03:42:17,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1681a3a0c47f42de9c690bb1db1169cc as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1681a3a0c47f42de9c690bb1db1169cc 2024-11-28T03:42:17,485 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1681a3a0c47f42de9c690bb1db1169cc, entries=150, sequenceid=721, filesize=12.0 K 2024-11-28T03:42:17,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/38de02a0168049bcbd8fb13ff1f6ca98 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/38de02a0168049bcbd8fb13ff1f6ca98 2024-11-28T03:42:17,489 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/38de02a0168049bcbd8fb13ff1f6ca98, entries=150, sequenceid=721, filesize=12.0 K 2024-11-28T03:42:17,490 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1a2c0de76d7e5c66916b7f6387695a75 in 873ms, sequenceid=721, compaction requested=true 2024-11-28T03:42:17,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:17,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:17,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-28T03:42:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-28T03:42:17,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-28T03:42:17,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0310 sec 2024-11-28T03:42:17,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.0370 sec 2024-11-28T03:42:17,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T03:42:17,596 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:17,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-28T03:42:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T03:42:17,599 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:17,599 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:17,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:17,604 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T03:42:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T03:42:17,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-28T03:42:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:17,756 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:42:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:17,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/9d5f7c6958c1476e93e318113efc3ea9 is 50, key is test_row_0/A:col10/1732765336735/Put/seqid=0 2024-11-28T03:42:17,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742181_1357 (size=12301) 2024-11-28T03:42:17,766 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=739 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/9d5f7c6958c1476e93e318113efc3ea9 2024-11-28T03:42:17,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/a628ac8302934e5090c5b1ae87a1300b is 50, key is test_row_0/B:col10/1732765336735/Put/seqid=0 2024-11-28T03:42:17,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742182_1358 (size=12301) 2024-11-28T03:42:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:17,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765397896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:17,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765397898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:17,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765397901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T03:42:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765398007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765398007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765398016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:18,189 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=739 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/a628ac8302934e5090c5b1ae87a1300b 2024-11-28T03:42:18,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/6d26e232855b47e2b3b62e1911849651 is 50, key is test_row_0/C:col10/1732765336735/Put/seqid=0 2024-11-28T03:42:18,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742183_1359 (size=12301) 2024-11-28T03:42:18,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765398217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:18,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:18,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765398217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:18,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T03:42:18,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765398226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:18,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765398526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:18,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765398527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:18,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765398536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:18,606 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=739 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/6d26e232855b47e2b3b62e1911849651 2024-11-28T03:42:18,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/9d5f7c6958c1476e93e318113efc3ea9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d5f7c6958c1476e93e318113efc3ea9 2024-11-28T03:42:18,614 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d5f7c6958c1476e93e318113efc3ea9, entries=150, sequenceid=739, filesize=12.0 K 2024-11-28T03:42:18,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/a628ac8302934e5090c5b1ae87a1300b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a628ac8302934e5090c5b1ae87a1300b 2024-11-28T03:42:18,619 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a628ac8302934e5090c5b1ae87a1300b, entries=150, sequenceid=739, filesize=12.0 K 2024-11-28T03:42:18,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/6d26e232855b47e2b3b62e1911849651 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6d26e232855b47e2b3b62e1911849651 2024-11-28T03:42:18,623 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6d26e232855b47e2b3b62e1911849651, entries=150, sequenceid=739, filesize=12.0 K 2024-11-28T03:42:18,624 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1a2c0de76d7e5c66916b7f6387695a75 in 868ms, sequenceid=739, compaction requested=true 2024-11-28T03:42:18,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:18,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:18,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-28T03:42:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-28T03:42:18,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-28T03:42:18,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0260 sec 2024-11-28T03:42:18,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.0310 sec 2024-11-28T03:42:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T03:42:18,737 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:18,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-28T03:42:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T03:42:18,739 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:18,740 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:18,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T03:42:18,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-28T03:42:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:18,893 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:42:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:18,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/919a92b3bb3044ceb9b19214139eb7b7 is 50, key is test_row_0/A:col10/1732765337899/Put/seqid=0 2024-11-28T03:42:18,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742184_1360 (size=12301) 2024-11-28T03:42:18,902 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=753 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/919a92b3bb3044ceb9b19214139eb7b7 2024-11-28T03:42:18,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/55792702437d4d1b8146a1f315b58069 is 50, key is test_row_0/B:col10/1732765337899/Put/seqid=0 2024-11-28T03:42:18,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742185_1361 (size=12301) 2024-11-28T03:42:18,919 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=753 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/55792702437d4d1b8146a1f315b58069 2024-11-28T03:42:18,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/8e26d63e04d74211b5f0bd41b82b211a is 50, key is test_row_0/C:col10/1732765337899/Put/seqid=0 2024-11-28T03:42:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742186_1362 (size=12301) 2024-11-28T03:42:18,962 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=753 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/8e26d63e04d74211b5f0bd41b82b211a 2024-11-28T03:42:18,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/919a92b3bb3044ceb9b19214139eb7b7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/919a92b3bb3044ceb9b19214139eb7b7 2024-11-28T03:42:18,977 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/919a92b3bb3044ceb9b19214139eb7b7, entries=150, sequenceid=753, filesize=12.0 K 2024-11-28T03:42:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/55792702437d4d1b8146a1f315b58069 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/55792702437d4d1b8146a1f315b58069 2024-11-28T03:42:18,984 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/55792702437d4d1b8146a1f315b58069, entries=150, sequenceid=753, filesize=12.0 K 2024-11-28T03:42:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/8e26d63e04d74211b5f0bd41b82b211a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8e26d63e04d74211b5f0bd41b82b211a 2024-11-28T03:42:18,989 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8e26d63e04d74211b5f0bd41b82b211a, entries=150, sequenceid=753, filesize=12.0 K 2024-11-28T03:42:18,990 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 1a2c0de76d7e5c66916b7f6387695a75 in 97ms, sequenceid=753, compaction requested=true 2024-11-28T03:42:18,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:18,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:18,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-28T03:42:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-28T03:42:18,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-28T03:42:18,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 253 msec 2024-11-28T03:42:18,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 260 msec 2024-11-28T03:42:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:19,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:42:19,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:19,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:19,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:19,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:19,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:19,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T03:42:19,057 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:19,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-28T03:42:19,060 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:19,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6f03c6b3789d4fac872dc8027dbfd4a7 is 50, key is test_row_0/A:col10/1732765339049/Put/seqid=0 2024-11-28T03:42:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-28T03:42:19,060 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:19,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:19,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742187_1363 (size=12301) 2024-11-28T03:42:19,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6f03c6b3789d4fac872dc8027dbfd4a7 2024-11-28T03:42:19,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/813589a2aaea42a58078758f627f5752 is 50, key is test_row_0/B:col10/1732765339049/Put/seqid=0 2024-11-28T03:42:19,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742188_1364 (size=12301) 2024-11-28T03:42:19,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/813589a2aaea42a58078758f627f5752 2024-11-28T03:42:19,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/6c3cea811b44438c80fa7b4c3786d33b is 50, key is test_row_0/C:col10/1732765339049/Put/seqid=0 2024-11-28T03:42:19,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765339062, max timestamp=1732765339109Num uniques 859; 2024-11-28T03:42:19,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:42:19,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765339062, max timestamp=1732765339109Num uniques 859; 2024-11-28T03:42:19,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:42:19,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765339062, max timestamp=1732765339109Num uniques 859; 2024-11-28T03:42:19,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:42:19,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742189_1365 (size=12301) 2024-11-28T03:42:19,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765399153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-28T03:42:19,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765399158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:19,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765399168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:19,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:19,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-28T03:42:19,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:19,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:19,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:19,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:19,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:19,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765399266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:19,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765399276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765399277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:19,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-28T03:42:19,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:19,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:19,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:19,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:19,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-28T03:42:19,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38414 deadline: 1732765399476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:19,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38364 deadline: 1732765399487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:19,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732765399487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:19,504 INFO [Thread-1183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1183. 2024-11-28T03:42:19,504 DEBUG [Thread-1183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:19,504 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,504 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:19,507 INFO [Thread-1181 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1181. 2024-11-28T03:42:19,507 DEBUG [Thread-1181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:19,507 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,507 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,507 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-11-28T03:42:19,507 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:19,508 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:19,508 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,508 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,508 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:19,511 INFO [Thread-1185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1185. 2024-11-28T03:42:19,511 DEBUG [Thread-1185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:19,511 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,511 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:19,513 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-11-28T03:42:19,513 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:19,513 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,513 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,513 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:19,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-28T03:42:19,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:19,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:19,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:19,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:19,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:19,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/6c3cea811b44438c80fa7b4c3786d33b 2024-11-28T03:42:19,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/6f03c6b3789d4fac872dc8027dbfd4a7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6f03c6b3789d4fac872dc8027dbfd4a7 2024-11-28T03:42:19,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6f03c6b3789d4fac872dc8027dbfd4a7, entries=150, sequenceid=763, filesize=12.0 K 2024-11-28T03:42:19,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/813589a2aaea42a58078758f627f5752 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/813589a2aaea42a58078758f627f5752 2024-11-28T03:42:19,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/813589a2aaea42a58078758f627f5752, entries=150, sequenceid=763, filesize=12.0 K 2024-11-28T03:42:19,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/6c3cea811b44438c80fa7b4c3786d33b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6c3cea811b44438c80fa7b4c3786d33b 2024-11-28T03:42:19,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6c3cea811b44438c80fa7b4c3786d33b, entries=150, sequenceid=763, filesize=12.0 K 2024-11-28T03:42:19,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1a2c0de76d7e5c66916b7f6387695a75 in 487ms, sequenceid=763, compaction requested=true 2024-11-28T03:42:19,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:19,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:19,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:19,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:19,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:19,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a2c0de76d7e5c66916b7f6387695a75:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:19,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:19,541 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-28T03:42:19,541 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-28T03:42:19,543 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 99430 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-28T03:42:19,543 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 99430 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-28T03:42:19,543 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/B is initiating minor compaction (all files) 2024-11-28T03:42:19,543 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/A is initiating minor compaction (all files) 2024-11-28T03:42:19,543 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/B in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:19,543 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/A in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:19,543 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/9136f7865f1d472580cc78b7a4cd5b02, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/839b7bd02d68414ca22affee8b7919b9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fb461f044eba41718ab69fc81c8b3c52, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58e0f906d8e1454bb0d0896e5ed64bcc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1681a3a0c47f42de9c690bb1db1169cc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a628ac8302934e5090c5b1ae87a1300b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/55792702437d4d1b8146a1f315b58069, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/813589a2aaea42a58078758f627f5752] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=97.1 K 2024-11-28T03:42:19,543 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/71c4b655cdad4df7b2c239c14900fb77, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/db6da8edb2284ab3a5ea6a9d87e9ab66, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/646851f40a9d498da569c65366aa15a8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/77d1ca1c2ed5450295c52bee45e10031, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6475f53d0688400797cb7300e55d82ac, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d5f7c6958c1476e93e318113efc3ea9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/919a92b3bb3044ceb9b19214139eb7b7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6f03c6b3789d4fac872dc8027dbfd4a7] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=97.1 K 2024-11-28T03:42:19,544 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9136f7865f1d472580cc78b7a4cd5b02, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1732765332516 2024-11-28T03:42:19,544 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c4b655cdad4df7b2c239c14900fb77, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1732765332516 2024-11-28T03:42:19,544 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 839b7bd02d68414ca22affee8b7919b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=674, earliestPutTs=1732765332550 2024-11-28T03:42:19,544 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting db6da8edb2284ab3a5ea6a9d87e9ab66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=674, earliestPutTs=1732765332550 2024-11-28T03:42:19,544 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting fb461f044eba41718ab69fc81c8b3c52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1732765333724 2024-11-28T03:42:19,544 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 646851f40a9d498da569c65366aa15a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1732765333724 2024-11-28T03:42:19,544 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e0f906d8e1454bb0d0896e5ed64bcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=707, earliestPutTs=1732765334370 2024-11-28T03:42:19,544 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1681a3a0c47f42de9c690bb1db1169cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=721, earliestPutTs=1732765335556 2024-11-28T03:42:19,544 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77d1ca1c2ed5450295c52bee45e10031, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=707, earliestPutTs=1732765334370 2024-11-28T03:42:19,545 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting a628ac8302934e5090c5b1ae87a1300b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=739, earliestPutTs=1732765336728 2024-11-28T03:42:19,545 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6475f53d0688400797cb7300e55d82ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=721, earliestPutTs=1732765335556 2024-11-28T03:42:19,545 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 55792702437d4d1b8146a1f315b58069, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=753, earliestPutTs=1732765337894 2024-11-28T03:42:19,545 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d5f7c6958c1476e93e318113efc3ea9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=739, earliestPutTs=1732765336728 2024-11-28T03:42:19,545 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 813589a2aaea42a58078758f627f5752, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1732765339049 2024-11-28T03:42:19,545 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 919a92b3bb3044ceb9b19214139eb7b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=753, earliestPutTs=1732765337894 2024-11-28T03:42:19,545 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f03c6b3789d4fac872dc8027dbfd4a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1732765339049 2024-11-28T03:42:19,558 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#B#compaction#321 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:19,558 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/7a1b68c0a61149afa46c7b16000db194 is 50, key is test_row_0/B:col10/1732765339049/Put/seqid=0 2024-11-28T03:42:19,559 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#A#compaction#322 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:19,560 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/780d34a34d2444c5b59b8367ca6ae1bc is 50, key is test_row_0/A:col10/1732765339049/Put/seqid=0 2024-11-28T03:42:19,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742190_1366 (size=13595) 2024-11-28T03:42:19,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742191_1367 (size=13595) 2024-11-28T03:42:19,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-28T03:42:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:19,676 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T03:42:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/0adc1915c8c8456ab1167ca8075ad5a0 is 50, key is test_row_0/A:col10/1732765339161/Put/seqid=0 2024-11-28T03:42:19,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742192_1368 (size=12301) 2024-11-28T03:42:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-28T03:42:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:19,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. as already flushing 2024-11-28T03:42:19,798 INFO [Thread-1174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1174. 2024-11-28T03:42:19,798 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-28T03:42:19,798 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-11-28T03:42:19,798 DEBUG [Thread-1174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:19,798 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,798 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:19,798 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:19,798 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,798 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,798 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,798 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,798 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:19,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:19,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:19,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:19,966 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/7a1b68c0a61149afa46c7b16000db194 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/7a1b68c0a61149afa46c7b16000db194 2024-11-28T03:42:19,967 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/780d34a34d2444c5b59b8367ca6ae1bc as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/780d34a34d2444c5b59b8367ca6ae1bc 2024-11-28T03:42:19,971 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/B of 1a2c0de76d7e5c66916b7f6387695a75 into 7a1b68c0a61149afa46c7b16000db194(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:19,971 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:19,971 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/B, priority=8, startTime=1732765339541; duration=0sec 2024-11-28T03:42:19,971 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:19,971 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:B 2024-11-28T03:42:19,971 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-28T03:42:19,971 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/A of 1a2c0de76d7e5c66916b7f6387695a75 into 780d34a34d2444c5b59b8367ca6ae1bc(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:19,971 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:19,971 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/A, priority=8, startTime=1732765339541; duration=0sec 2024-11-28T03:42:19,971 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:19,971 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:A 2024-11-28T03:42:19,972 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 99430 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-28T03:42:19,972 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 1a2c0de76d7e5c66916b7f6387695a75/C is initiating minor compaction (all files) 2024-11-28T03:42:19,972 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a2c0de76d7e5c66916b7f6387695a75/C in TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:19,972 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/4573bd04c48f4f06b5d39624a5a417d6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3a1717d8d35548a5be3da3aeefdaca51, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8c195c479c064bee972368fbc9f3512f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ec85dd38c1634a65a5a825752ffa3224, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/38de02a0168049bcbd8fb13ff1f6ca98, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6d26e232855b47e2b3b62e1911849651, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8e26d63e04d74211b5f0bd41b82b211a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6c3cea811b44438c80fa7b4c3786d33b] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp, totalSize=97.1 K 2024-11-28T03:42:19,973 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4573bd04c48f4f06b5d39624a5a417d6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1732765332516 2024-11-28T03:42:19,973 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a1717d8d35548a5be3da3aeefdaca51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=674, earliestPutTs=1732765332550 2024-11-28T03:42:19,973 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c195c479c064bee972368fbc9f3512f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1732765333724 2024-11-28T03:42:19,973 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ec85dd38c1634a65a5a825752ffa3224, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=707, earliestPutTs=1732765334370 2024-11-28T03:42:19,973 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 38de02a0168049bcbd8fb13ff1f6ca98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=721, earliestPutTs=1732765335556 2024-11-28T03:42:19,974 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d26e232855b47e2b3b62e1911849651, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=739, earliestPutTs=1732765336728 2024-11-28T03:42:19,974 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e26d63e04d74211b5f0bd41b82b211a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=753, earliestPutTs=1732765337894 2024-11-28T03:42:19,974 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c3cea811b44438c80fa7b4c3786d33b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1732765339049 2024-11-28T03:42:19,984 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a2c0de76d7e5c66916b7f6387695a75#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:19,985 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/66f637806069442ebd8b2e309c36cee1 is 50, key is test_row_0/C:col10/1732765339049/Put/seqid=0 2024-11-28T03:42:19,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742193_1369 (size=13595) 2024-11-28T03:42:20,084 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=799 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/0adc1915c8c8456ab1167ca8075ad5a0 2024-11-28T03:42:20,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1d536e245a7a408292f5200e20ace058 is 50, key is test_row_0/B:col10/1732765339161/Put/seqid=0 2024-11-28T03:42:20,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742194_1370 (size=12301) 2024-11-28T03:42:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-28T03:42:20,392 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/66f637806069442ebd8b2e309c36cee1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/66f637806069442ebd8b2e309c36cee1 2024-11-28T03:42:20,396 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 1a2c0de76d7e5c66916b7f6387695a75/C of 1a2c0de76d7e5c66916b7f6387695a75 into 66f637806069442ebd8b2e309c36cee1(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:20,396 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:20,396 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., storeName=1a2c0de76d7e5c66916b7f6387695a75/C, priority=8, startTime=1732765339541; duration=0sec 2024-11-28T03:42:20,396 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:20,396 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a2c0de76d7e5c66916b7f6387695a75:C 2024-11-28T03:42:20,494 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=799 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1d536e245a7a408292f5200e20ace058 2024-11-28T03:42:20,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/343c062cfd8342a185aff2edca233773 is 50, key is test_row_0/C:col10/1732765339161/Put/seqid=0 2024-11-28T03:42:20,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742195_1371 (size=12301) 2024-11-28T03:42:20,905 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=799 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/343c062cfd8342a185aff2edca233773 2024-11-28T03:42:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/0adc1915c8c8456ab1167ca8075ad5a0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/0adc1915c8c8456ab1167ca8075ad5a0 2024-11-28T03:42:20,913 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/0adc1915c8c8456ab1167ca8075ad5a0, entries=150, sequenceid=799, filesize=12.0 K 2024-11-28T03:42:20,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/1d536e245a7a408292f5200e20ace058 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1d536e245a7a408292f5200e20ace058 2024-11-28T03:42:20,916 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1d536e245a7a408292f5200e20ace058, entries=150, sequenceid=799, filesize=12.0 K 2024-11-28T03:42:20,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/343c062cfd8342a185aff2edca233773 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/343c062cfd8342a185aff2edca233773 2024-11-28T03:42:20,920 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/343c062cfd8342a185aff2edca233773, entries=150, sequenceid=799, filesize=12.0 K 2024-11-28T03:42:20,921 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=20.13 KB/20610 for 1a2c0de76d7e5c66916b7f6387695a75 in 1245ms, sequenceid=799, compaction requested=false 2024-11-28T03:42:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 1a2c0de76d7e5c66916b7f6387695a75: 2024-11-28T03:42:20,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:20,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-28T03:42:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-28T03:42:20,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-28T03:42:20,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8620 sec 2024-11-28T03:42:20,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.8670 sec 2024-11-28T03:42:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-28T03:42:21,207 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:23,039 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-28T03:42:23,039 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:23,039 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:23,039 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:23,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:23,099 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-28T03:42:23,099 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:23,099 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:23,099 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:23,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:23,099 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T03:42:23,099 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-28T03:42:23,099 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 222 2024-11-28T03:42:23,099 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-28T03:42:23,099 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 203 2024-11-28T03:42:23,099 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 184 2024-11-28T03:42:23,099 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T03:42:23,099 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2640 2024-11-28T03:42:23,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2643 2024-11-28T03:42:23,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2582 2024-11-28T03:42:23,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2640 2024-11-28T03:42:23,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2670 2024-11-28T03:42:23,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T03:42:23,100 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:42:23,100 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:42:23,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:23,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:23,100 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:23,101 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T03:42:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-28T03:42:23,107 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765343107"}]},"ts":"1732765343107"} 2024-11-28T03:42:23,109 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T03:42:23,109 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T03:42:23,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T03:42:23,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a2c0de76d7e5c66916b7f6387695a75, UNASSIGN}] 2024-11-28T03:42:23,113 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a2c0de76d7e5c66916b7f6387695a75, UNASSIGN 2024-11-28T03:42:23,114 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=1a2c0de76d7e5c66916b7f6387695a75, regionState=CLOSING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:42:23,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a2c0de76d7e5c66916b7f6387695a75, UNASSIGN because future has completed 2024-11-28T03:42:23,115 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T03:42:23,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:42:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-28T03:42:23,268 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:23,268 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T03:42:23,268 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 1a2c0de76d7e5c66916b7f6387695a75, disabling compactions & flushes 2024-11-28T03:42:23,269 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:23,269 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:23,269 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. after waiting 0 ms 2024-11-28T03:42:23,269 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:23,269 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing 1a2c0de76d7e5c66916b7f6387695a75 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T03:42:23,269 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=A 2024-11-28T03:42:23,269 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:23,269 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=B 2024-11-28T03:42:23,269 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:23,269 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a2c0de76d7e5c66916b7f6387695a75, store=C 2024-11-28T03:42:23,269 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:23,273 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/532ef9e71d904b1fbb8d8a2e4c5a5c68 is 50, key is test_row_0/A:col10/1732765343037/Put/seqid=0 2024-11-28T03:42:23,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742196_1372 (size=12301) 2024-11-28T03:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-28T03:42:23,677 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=810 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/532ef9e71d904b1fbb8d8a2e4c5a5c68 2024-11-28T03:42:23,683 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/f73de102fbd6447797db31daeaecde9f is 50, key is test_row_0/B:col10/1732765343037/Put/seqid=0 2024-11-28T03:42:23,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742197_1373 (size=12301) 2024-11-28T03:42:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-28T03:42:24,087 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=810 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/f73de102fbd6447797db31daeaecde9f 2024-11-28T03:42:24,093 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/bf8775f95fdd4708b7cdf82552cb5edc is 50, key is test_row_0/C:col10/1732765343037/Put/seqid=0 2024-11-28T03:42:24,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742198_1374 (size=12301) 2024-11-28T03:42:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-28T03:42:24,497 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=810 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/bf8775f95fdd4708b7cdf82552cb5edc 2024-11-28T03:42:24,501 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/A/532ef9e71d904b1fbb8d8a2e4c5a5c68 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/532ef9e71d904b1fbb8d8a2e4c5a5c68 2024-11-28T03:42:24,504 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/532ef9e71d904b1fbb8d8a2e4c5a5c68, entries=150, sequenceid=810, filesize=12.0 K 2024-11-28T03:42:24,504 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/B/f73de102fbd6447797db31daeaecde9f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f73de102fbd6447797db31daeaecde9f 2024-11-28T03:42:24,507 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f73de102fbd6447797db31daeaecde9f, entries=150, sequenceid=810, filesize=12.0 K 2024-11-28T03:42:24,508 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/.tmp/C/bf8775f95fdd4708b7cdf82552cb5edc as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/bf8775f95fdd4708b7cdf82552cb5edc 2024-11-28T03:42:24,510 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/bf8775f95fdd4708b7cdf82552cb5edc, entries=150, sequenceid=810, filesize=12.0 K 2024-11-28T03:42:24,511 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1a2c0de76d7e5c66916b7f6387695a75 in 1242ms, sequenceid=810, compaction requested=true 2024-11-28T03:42:24,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/ede0dfa327c04956a6499ab590513518, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/95d4314e584c48809ec27b156c581336, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e3e989e361f1413eb21e3c6f0093c34e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/438708826f54453eb8ad947d609a2901, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1983bafceb1d4da4ab88846b8b28fd30, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9a1f1740bcfe4313aa927725983b2c26, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e87a56b9725247ab8a7f7a1cf21a9a27, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/2ca894b13f4e41afa75dbbfb8d9097d4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/92c54c0d929f4f88b3e5c0af418c9a21, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/07e82603db8547eda7f112fad6c5fed4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d6eeaf247ad4564a4dfbf16176c083e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6e7888e4d5954f71a5b6efb2255f2237, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/638993eef20c422b9798b510e48dfc07, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/a38974465307498c9a13e705769cca3a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9f5b85a788cd4c2696ebc5450725f9d2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/aff0cd7e153a41f4be55f37a1f62ea6c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/8cc1613699a0496cb24837daff198152, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6be8c87b97e148ac91afaf8d04d2e29b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/c75146c90fc64d3f9a554fa7021b1c59, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4e61fbf0161e4b32969b0a911b2d168a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/f6973eb2f5e34f368d36b7a916da9a33, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/223677918d044d13bf7409b5ab61c5d1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4579b60c08724b3aa82a4f54d00d6aad, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1342a1867dfe4caf89934596d01afa56, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/41c2b61660d241ae9cd624a057b3312e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/cd135b8c46b64d0091ca5470792c15db, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/72cdb121f04743e7ac4b85de9d9650a2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/d90b0e90831945d9b9a4f4a7ac7771cd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/dbb29544708845f2b570b0ed2cf71551, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4aec824ed01b45eb9da9c2eb541467c9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/71c4b655cdad4df7b2c239c14900fb77, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/db6da8edb2284ab3a5ea6a9d87e9ab66, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/646851f40a9d498da569c65366aa15a8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/77d1ca1c2ed5450295c52bee45e10031, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6475f53d0688400797cb7300e55d82ac, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d5f7c6958c1476e93e318113efc3ea9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/919a92b3bb3044ceb9b19214139eb7b7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6f03c6b3789d4fac872dc8027dbfd4a7] to archive 2024-11-28T03:42:24,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:42:24,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/ede0dfa327c04956a6499ab590513518 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/ede0dfa327c04956a6499ab590513518 2024-11-28T03:42:24,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/95d4314e584c48809ec27b156c581336 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/95d4314e584c48809ec27b156c581336 2024-11-28T03:42:24,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e3e989e361f1413eb21e3c6f0093c34e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e3e989e361f1413eb21e3c6f0093c34e 2024-11-28T03:42:24,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/438708826f54453eb8ad947d609a2901 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/438708826f54453eb8ad947d609a2901 2024-11-28T03:42:24,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1983bafceb1d4da4ab88846b8b28fd30 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1983bafceb1d4da4ab88846b8b28fd30 2024-11-28T03:42:24,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9a1f1740bcfe4313aa927725983b2c26 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9a1f1740bcfe4313aa927725983b2c26 2024-11-28T03:42:24,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e87a56b9725247ab8a7f7a1cf21a9a27 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/e87a56b9725247ab8a7f7a1cf21a9a27 2024-11-28T03:42:24,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/2ca894b13f4e41afa75dbbfb8d9097d4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/2ca894b13f4e41afa75dbbfb8d9097d4 2024-11-28T03:42:24,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/92c54c0d929f4f88b3e5c0af418c9a21 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/92c54c0d929f4f88b3e5c0af418c9a21 2024-11-28T03:42:24,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/07e82603db8547eda7f112fad6c5fed4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/07e82603db8547eda7f112fad6c5fed4 2024-11-28T03:42:24,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d6eeaf247ad4564a4dfbf16176c083e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d6eeaf247ad4564a4dfbf16176c083e 2024-11-28T03:42:24,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6e7888e4d5954f71a5b6efb2255f2237 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6e7888e4d5954f71a5b6efb2255f2237 2024-11-28T03:42:24,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/638993eef20c422b9798b510e48dfc07 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/638993eef20c422b9798b510e48dfc07 2024-11-28T03:42:24,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/a38974465307498c9a13e705769cca3a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/a38974465307498c9a13e705769cca3a 2024-11-28T03:42:24,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9f5b85a788cd4c2696ebc5450725f9d2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9f5b85a788cd4c2696ebc5450725f9d2 2024-11-28T03:42:24,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/aff0cd7e153a41f4be55f37a1f62ea6c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/aff0cd7e153a41f4be55f37a1f62ea6c 2024-11-28T03:42:24,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/8cc1613699a0496cb24837daff198152 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/8cc1613699a0496cb24837daff198152 2024-11-28T03:42:24,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6be8c87b97e148ac91afaf8d04d2e29b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6be8c87b97e148ac91afaf8d04d2e29b 2024-11-28T03:42:24,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/c75146c90fc64d3f9a554fa7021b1c59 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/c75146c90fc64d3f9a554fa7021b1c59 2024-11-28T03:42:24,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4e61fbf0161e4b32969b0a911b2d168a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4e61fbf0161e4b32969b0a911b2d168a 2024-11-28T03:42:24,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/f6973eb2f5e34f368d36b7a916da9a33 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/f6973eb2f5e34f368d36b7a916da9a33 2024-11-28T03:42:24,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/223677918d044d13bf7409b5ab61c5d1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/223677918d044d13bf7409b5ab61c5d1 2024-11-28T03:42:24,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4579b60c08724b3aa82a4f54d00d6aad to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4579b60c08724b3aa82a4f54d00d6aad 2024-11-28T03:42:24,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1342a1867dfe4caf89934596d01afa56 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/1342a1867dfe4caf89934596d01afa56 2024-11-28T03:42:24,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/41c2b61660d241ae9cd624a057b3312e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/41c2b61660d241ae9cd624a057b3312e 2024-11-28T03:42:24,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/cd135b8c46b64d0091ca5470792c15db to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/cd135b8c46b64d0091ca5470792c15db 2024-11-28T03:42:24,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/72cdb121f04743e7ac4b85de9d9650a2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/72cdb121f04743e7ac4b85de9d9650a2 2024-11-28T03:42:24,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/d90b0e90831945d9b9a4f4a7ac7771cd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/d90b0e90831945d9b9a4f4a7ac7771cd 2024-11-28T03:42:24,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/dbb29544708845f2b570b0ed2cf71551 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/dbb29544708845f2b570b0ed2cf71551 2024-11-28T03:42:24,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4aec824ed01b45eb9da9c2eb541467c9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/4aec824ed01b45eb9da9c2eb541467c9 2024-11-28T03:42:24,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/71c4b655cdad4df7b2c239c14900fb77 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/71c4b655cdad4df7b2c239c14900fb77 2024-11-28T03:42:24,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/db6da8edb2284ab3a5ea6a9d87e9ab66 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/db6da8edb2284ab3a5ea6a9d87e9ab66 2024-11-28T03:42:24,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/646851f40a9d498da569c65366aa15a8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/646851f40a9d498da569c65366aa15a8 2024-11-28T03:42:24,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/77d1ca1c2ed5450295c52bee45e10031 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/77d1ca1c2ed5450295c52bee45e10031 2024-11-28T03:42:24,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6475f53d0688400797cb7300e55d82ac to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6475f53d0688400797cb7300e55d82ac 2024-11-28T03:42:24,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d5f7c6958c1476e93e318113efc3ea9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/9d5f7c6958c1476e93e318113efc3ea9 2024-11-28T03:42:24,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/919a92b3bb3044ceb9b19214139eb7b7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/919a92b3bb3044ceb9b19214139eb7b7 2024-11-28T03:42:24,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6f03c6b3789d4fac872dc8027dbfd4a7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/6f03c6b3789d4fac872dc8027dbfd4a7 2024-11-28T03:42:24,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c11430cbddec4f7b8b1a0fbe92cadf80, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9e30b233569428a94c4904780a99ec2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/865916afec474c8391700cbaea3709e8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58445c9b179d4496bd2793f9228838a0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c6c5b54975864f36b8965ef60ca5ee68, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9db127ba09e4df5b5ec9975b0ad4293, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8519e9db8dc3439db3be13095a49f666, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74fab4430c2c4adfac77fb47dd1ddd88, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c5a7f7cdabbb40fe9de3e43ab1660cab, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/83720ad7a3444cacae4a217c9d72e1ca, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/36688c827c614181811d077b16ba8540, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f57ae1123a384198ad10f35b52972322, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74b4041c533c44aeb5fa2e305975e2d4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3cb4cf3ff88b42a9b9e2b004080f58c5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3184713b908843d29aa1ea177058d620, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/bc722f1851b5441aa667ccf193ff1a17, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1226dd853da04f018360022c2601d633, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/041f2c99e80345458caa0bcc82db3a45, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/ec9ce3cfb0634abfb848bef103e2cdba, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/0a3eecb886e844b381f19465ed00cfb4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/27d175931eb44fcbb2c9a8d6ef4c5cff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/981034beccbd4bfcba8efc3e226128ef, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a134157017dc4b3f9f7cfeb23114bdc1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f4f506ce0d894074b0b81313db2053f7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/2f41de79b0804763bc21acfe7ae3f7a7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1e62516a7f334e95aa09391b31b497ed, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8e894f3d0e294f969eb123a6989c563e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/70c1636d5cff4ec5b9c06380575d75a7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fbb9352a2b0745a78667e1bb0b968092, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/9136f7865f1d472580cc78b7a4cd5b02, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/759f16691344423394f7ea1247232135, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/839b7bd02d68414ca22affee8b7919b9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fb461f044eba41718ab69fc81c8b3c52, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58e0f906d8e1454bb0d0896e5ed64bcc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1681a3a0c47f42de9c690bb1db1169cc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a628ac8302934e5090c5b1ae87a1300b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/55792702437d4d1b8146a1f315b58069, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/813589a2aaea42a58078758f627f5752] to archive 2024-11-28T03:42:24,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:42:24,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c11430cbddec4f7b8b1a0fbe92cadf80 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c11430cbddec4f7b8b1a0fbe92cadf80 2024-11-28T03:42:24,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9e30b233569428a94c4904780a99ec2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9e30b233569428a94c4904780a99ec2 2024-11-28T03:42:24,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/865916afec474c8391700cbaea3709e8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/865916afec474c8391700cbaea3709e8 2024-11-28T03:42:24,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58445c9b179d4496bd2793f9228838a0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58445c9b179d4496bd2793f9228838a0 2024-11-28T03:42:24,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c6c5b54975864f36b8965ef60ca5ee68 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c6c5b54975864f36b8965ef60ca5ee68 2024-11-28T03:42:24,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9db127ba09e4df5b5ec9975b0ad4293 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/b9db127ba09e4df5b5ec9975b0ad4293 2024-11-28T03:42:24,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8519e9db8dc3439db3be13095a49f666 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8519e9db8dc3439db3be13095a49f666 2024-11-28T03:42:24,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74fab4430c2c4adfac77fb47dd1ddd88 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74fab4430c2c4adfac77fb47dd1ddd88 2024-11-28T03:42:24,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c5a7f7cdabbb40fe9de3e43ab1660cab to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/c5a7f7cdabbb40fe9de3e43ab1660cab 2024-11-28T03:42:24,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/83720ad7a3444cacae4a217c9d72e1ca to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/83720ad7a3444cacae4a217c9d72e1ca 2024-11-28T03:42:24,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/36688c827c614181811d077b16ba8540 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/36688c827c614181811d077b16ba8540 2024-11-28T03:42:24,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f57ae1123a384198ad10f35b52972322 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f57ae1123a384198ad10f35b52972322 2024-11-28T03:42:24,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74b4041c533c44aeb5fa2e305975e2d4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/74b4041c533c44aeb5fa2e305975e2d4 2024-11-28T03:42:24,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3cb4cf3ff88b42a9b9e2b004080f58c5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3cb4cf3ff88b42a9b9e2b004080f58c5 2024-11-28T03:42:24,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3184713b908843d29aa1ea177058d620 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/3184713b908843d29aa1ea177058d620 2024-11-28T03:42:24,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/bc722f1851b5441aa667ccf193ff1a17 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/bc722f1851b5441aa667ccf193ff1a17 2024-11-28T03:42:24,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1226dd853da04f018360022c2601d633 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1226dd853da04f018360022c2601d633 2024-11-28T03:42:24,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/041f2c99e80345458caa0bcc82db3a45 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/041f2c99e80345458caa0bcc82db3a45 2024-11-28T03:42:24,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/ec9ce3cfb0634abfb848bef103e2cdba to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/ec9ce3cfb0634abfb848bef103e2cdba 2024-11-28T03:42:24,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/0a3eecb886e844b381f19465ed00cfb4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/0a3eecb886e844b381f19465ed00cfb4 2024-11-28T03:42:24,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/27d175931eb44fcbb2c9a8d6ef4c5cff to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/27d175931eb44fcbb2c9a8d6ef4c5cff 2024-11-28T03:42:24,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/981034beccbd4bfcba8efc3e226128ef to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/981034beccbd4bfcba8efc3e226128ef 2024-11-28T03:42:24,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a134157017dc4b3f9f7cfeb23114bdc1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a134157017dc4b3f9f7cfeb23114bdc1 2024-11-28T03:42:24,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f4f506ce0d894074b0b81313db2053f7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f4f506ce0d894074b0b81313db2053f7 2024-11-28T03:42:24,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/2f41de79b0804763bc21acfe7ae3f7a7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/2f41de79b0804763bc21acfe7ae3f7a7 2024-11-28T03:42:24,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1e62516a7f334e95aa09391b31b497ed to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1e62516a7f334e95aa09391b31b497ed 2024-11-28T03:42:24,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8e894f3d0e294f969eb123a6989c563e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/8e894f3d0e294f969eb123a6989c563e 2024-11-28T03:42:24,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/70c1636d5cff4ec5b9c06380575d75a7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/70c1636d5cff4ec5b9c06380575d75a7 2024-11-28T03:42:24,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fbb9352a2b0745a78667e1bb0b968092 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fbb9352a2b0745a78667e1bb0b968092 2024-11-28T03:42:24,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/9136f7865f1d472580cc78b7a4cd5b02 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/9136f7865f1d472580cc78b7a4cd5b02 2024-11-28T03:42:24,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/759f16691344423394f7ea1247232135 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/759f16691344423394f7ea1247232135 2024-11-28T03:42:24,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/839b7bd02d68414ca22affee8b7919b9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/839b7bd02d68414ca22affee8b7919b9 2024-11-28T03:42:24,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fb461f044eba41718ab69fc81c8b3c52 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/fb461f044eba41718ab69fc81c8b3c52 2024-11-28T03:42:24,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58e0f906d8e1454bb0d0896e5ed64bcc to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/58e0f906d8e1454bb0d0896e5ed64bcc 2024-11-28T03:42:24,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1681a3a0c47f42de9c690bb1db1169cc to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1681a3a0c47f42de9c690bb1db1169cc 2024-11-28T03:42:24,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a628ac8302934e5090c5b1ae87a1300b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/a628ac8302934e5090c5b1ae87a1300b 2024-11-28T03:42:24,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/55792702437d4d1b8146a1f315b58069 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/55792702437d4d1b8146a1f315b58069 2024-11-28T03:42:24,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/813589a2aaea42a58078758f627f5752 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/813589a2aaea42a58078758f627f5752 2024-11-28T03:42:24,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ab304b2a3e2940b7b174abf791e615b8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/7049c5c085804a5a9b8c59ff3fe7e22e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/5b74a4dda976496a8f22b569a9ea6149, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/557699d27d5d4846a84c1508e3532b2e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/db5e710b90be4974999099bb1a521018, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/16f0534246d647d18bc95a4e95e63ec9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3b32bdd5f26f4cb0900dcdd27e58f18b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/770b960c02ab4b76886ccc807aedd8f8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/5792ce52a136421fa834cb8da2e02e6b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9f3974ba809940da90bc4a083ed1621f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9de09106a95c41899a567894af2228d8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/d652de2c3f1e4d54b61c85725d3951d8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/664722a7215d4e5591a103eb8f5d5b01, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9fddd83d4cc94e3cba2cb4147971ea58, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/c95b538f65984809918e52a5ac6955ba, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/813b9eefc43145ec9822db1cae4a7cd3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/44abd3f9652c4f32990b14c30199d5e0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/18b453cf7faa483e9b320c76befdea8a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b9d81f0b1b1c4851a118a9111d76635b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8117a747d3f345f8b770a7c641c3b023, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/151fb73eef464b4d9ef326ace27bfb77, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/dbbb384329cc4dba9cc0b39a221e9b83, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/bb8a48593b1046e6800266d80636c834, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/a970bc56c04d4d3a99aea4f457dc8c8a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/e7f92a7f02354dafbb28be97fb90c799, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/eec441ecd20e46dfb1f2bdece26a96d8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3971718be5fe4d46808cb79d00901c7f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/abaa4999e9ee43108657dcdbe4554bf9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/206abc76834349d080666bad8c6bc7c5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/4573bd04c48f4f06b5d39624a5a417d6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b8e4da13084a4926848ba96ad5c97d27, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3a1717d8d35548a5be3da3aeefdaca51, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8c195c479c064bee972368fbc9f3512f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ec85dd38c1634a65a5a825752ffa3224, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/38de02a0168049bcbd8fb13ff1f6ca98, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6d26e232855b47e2b3b62e1911849651, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8e26d63e04d74211b5f0bd41b82b211a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6c3cea811b44438c80fa7b4c3786d33b] to archive 2024-11-28T03:42:24,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:42:24,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ab304b2a3e2940b7b174abf791e615b8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ab304b2a3e2940b7b174abf791e615b8 2024-11-28T03:42:24,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/7049c5c085804a5a9b8c59ff3fe7e22e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/7049c5c085804a5a9b8c59ff3fe7e22e 2024-11-28T03:42:24,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/5b74a4dda976496a8f22b569a9ea6149 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/5b74a4dda976496a8f22b569a9ea6149 2024-11-28T03:42:24,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/557699d27d5d4846a84c1508e3532b2e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/557699d27d5d4846a84c1508e3532b2e 2024-11-28T03:42:24,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/db5e710b90be4974999099bb1a521018 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/db5e710b90be4974999099bb1a521018 2024-11-28T03:42:24,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/16f0534246d647d18bc95a4e95e63ec9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/16f0534246d647d18bc95a4e95e63ec9 2024-11-28T03:42:24,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3b32bdd5f26f4cb0900dcdd27e58f18b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3b32bdd5f26f4cb0900dcdd27e58f18b 2024-11-28T03:42:24,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/770b960c02ab4b76886ccc807aedd8f8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/770b960c02ab4b76886ccc807aedd8f8 2024-11-28T03:42:24,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/5792ce52a136421fa834cb8da2e02e6b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/5792ce52a136421fa834cb8da2e02e6b 2024-11-28T03:42:24,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9f3974ba809940da90bc4a083ed1621f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9f3974ba809940da90bc4a083ed1621f 2024-11-28T03:42:24,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9de09106a95c41899a567894af2228d8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9de09106a95c41899a567894af2228d8 2024-11-28T03:42:24,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/d652de2c3f1e4d54b61c85725d3951d8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/d652de2c3f1e4d54b61c85725d3951d8 2024-11-28T03:42:24,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/664722a7215d4e5591a103eb8f5d5b01 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/664722a7215d4e5591a103eb8f5d5b01 2024-11-28T03:42:24,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9fddd83d4cc94e3cba2cb4147971ea58 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/9fddd83d4cc94e3cba2cb4147971ea58 2024-11-28T03:42:24,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/c95b538f65984809918e52a5ac6955ba to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/c95b538f65984809918e52a5ac6955ba 2024-11-28T03:42:24,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/813b9eefc43145ec9822db1cae4a7cd3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/813b9eefc43145ec9822db1cae4a7cd3 2024-11-28T03:42:24,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/44abd3f9652c4f32990b14c30199d5e0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/44abd3f9652c4f32990b14c30199d5e0 2024-11-28T03:42:24,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/18b453cf7faa483e9b320c76befdea8a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/18b453cf7faa483e9b320c76befdea8a 2024-11-28T03:42:24,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b9d81f0b1b1c4851a118a9111d76635b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b9d81f0b1b1c4851a118a9111d76635b 2024-11-28T03:42:24,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8117a747d3f345f8b770a7c641c3b023 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8117a747d3f345f8b770a7c641c3b023 2024-11-28T03:42:24,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/151fb73eef464b4d9ef326ace27bfb77 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/151fb73eef464b4d9ef326ace27bfb77 2024-11-28T03:42:24,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/dbbb384329cc4dba9cc0b39a221e9b83 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/dbbb384329cc4dba9cc0b39a221e9b83 2024-11-28T03:42:24,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/bb8a48593b1046e6800266d80636c834 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/bb8a48593b1046e6800266d80636c834 2024-11-28T03:42:24,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/a970bc56c04d4d3a99aea4f457dc8c8a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/a970bc56c04d4d3a99aea4f457dc8c8a 2024-11-28T03:42:24,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/e7f92a7f02354dafbb28be97fb90c799 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/e7f92a7f02354dafbb28be97fb90c799 2024-11-28T03:42:24,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/eec441ecd20e46dfb1f2bdece26a96d8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/eec441ecd20e46dfb1f2bdece26a96d8 2024-11-28T03:42:24,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3971718be5fe4d46808cb79d00901c7f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3971718be5fe4d46808cb79d00901c7f 2024-11-28T03:42:24,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/abaa4999e9ee43108657dcdbe4554bf9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/abaa4999e9ee43108657dcdbe4554bf9 2024-11-28T03:42:24,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/206abc76834349d080666bad8c6bc7c5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/206abc76834349d080666bad8c6bc7c5 2024-11-28T03:42:24,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/4573bd04c48f4f06b5d39624a5a417d6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/4573bd04c48f4f06b5d39624a5a417d6 2024-11-28T03:42:24,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b8e4da13084a4926848ba96ad5c97d27 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/b8e4da13084a4926848ba96ad5c97d27 2024-11-28T03:42:24,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3a1717d8d35548a5be3da3aeefdaca51 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/3a1717d8d35548a5be3da3aeefdaca51 2024-11-28T03:42:24,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8c195c479c064bee972368fbc9f3512f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8c195c479c064bee972368fbc9f3512f 2024-11-28T03:42:24,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ec85dd38c1634a65a5a825752ffa3224 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/ec85dd38c1634a65a5a825752ffa3224 2024-11-28T03:42:24,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/38de02a0168049bcbd8fb13ff1f6ca98 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/38de02a0168049bcbd8fb13ff1f6ca98 2024-11-28T03:42:24,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6d26e232855b47e2b3b62e1911849651 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6d26e232855b47e2b3b62e1911849651 2024-11-28T03:42:24,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8e26d63e04d74211b5f0bd41b82b211a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/8e26d63e04d74211b5f0bd41b82b211a 2024-11-28T03:42:24,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6c3cea811b44438c80fa7b4c3786d33b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/6c3cea811b44438c80fa7b4c3786d33b 2024-11-28T03:42:24,620 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/recovered.edits/813.seqid, newMaxSeqId=813, maxSeqId=1 2024-11-28T03:42:24,621 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. 2024-11-28T03:42:24,621 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 1a2c0de76d7e5c66916b7f6387695a75: Waiting for close lock at 1732765343268Running coprocessor pre-close hooks at 1732765343268Disabling compacts and flushes for region at 1732765343268Disabling writes for close at 1732765343269 (+1 ms)Obtaining lock to block concurrent updates at 1732765343269Preparing flush snapshotting stores in 1a2c0de76d7e5c66916b7f6387695a75 at 1732765343269Finished memstore snapshotting TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732765343269Flushing stores of TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75. at 1732765343270 (+1 ms)Flushing 1a2c0de76d7e5c66916b7f6387695a75/A: creating writer at 1732765343270Flushing 1a2c0de76d7e5c66916b7f6387695a75/A: appending metadata at 1732765343272 (+2 ms)Flushing 1a2c0de76d7e5c66916b7f6387695a75/A: closing flushed file at 1732765343272Flushing 1a2c0de76d7e5c66916b7f6387695a75/B: creating writer at 1732765343680 (+408 ms)Flushing 1a2c0de76d7e5c66916b7f6387695a75/B: appending metadata at 1732765343683 (+3 ms)Flushing 1a2c0de76d7e5c66916b7f6387695a75/B: closing flushed file at 1732765343683Flushing 1a2c0de76d7e5c66916b7f6387695a75/C: creating writer at 1732765344090 (+407 ms)Flushing 1a2c0de76d7e5c66916b7f6387695a75/C: appending metadata at 1732765344092 (+2 ms)Flushing 1a2c0de76d7e5c66916b7f6387695a75/C: closing flushed file at 1732765344093 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cba830b: reopening flushed file at 1732765344500 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13390017: reopening flushed file at 1732765344504 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14c0b456: reopening flushed file at 1732765344507 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1a2c0de76d7e5c66916b7f6387695a75 in 1242ms, sequenceid=810, compaction requested=true at 1732765344511 (+4 ms)Writing region close event to WAL at 1732765344617 (+106 ms)Running coprocessor post-close hooks at 1732765344620 (+3 ms)Closed at 1732765344620 2024-11-28T03:42:24,623 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:24,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=1a2c0de76d7e5c66916b7f6387695a75, regionState=CLOSED 2024-11-28T03:42:24,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:42:24,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-28T03:42:24,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a2c0de76d7e5c66916b7f6387695a75, server=3a546af21488,34671,1732765249447 in 1.5090 sec 2024-11-28T03:42:24,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-28T03:42:24,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a2c0de76d7e5c66916b7f6387695a75, UNASSIGN in 1.5140 sec 2024-11-28T03:42:24,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-28T03:42:24,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5180 sec 2024-11-28T03:42:24,631 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765344631"}]},"ts":"1732765344631"} 2024-11-28T03:42:24,632 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T03:42:24,632 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T03:42:24,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-28T03:42:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-28T03:42:25,257 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:25,258 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T03:42:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:25,259 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-28T03:42:25,260 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:25,262 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:25,263 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/recovered.edits] 2024-11-28T03:42:25,265 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/0adc1915c8c8456ab1167ca8075ad5a0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/0adc1915c8c8456ab1167ca8075ad5a0 2024-11-28T03:42:25,266 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/532ef9e71d904b1fbb8d8a2e4c5a5c68 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/532ef9e71d904b1fbb8d8a2e4c5a5c68 2024-11-28T03:42:25,267 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/780d34a34d2444c5b59b8367ca6ae1bc to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/A/780d34a34d2444c5b59b8367ca6ae1bc 2024-11-28T03:42:25,269 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1d536e245a7a408292f5200e20ace058 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/1d536e245a7a408292f5200e20ace058 2024-11-28T03:42:25,270 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/7a1b68c0a61149afa46c7b16000db194 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/7a1b68c0a61149afa46c7b16000db194 2024-11-28T03:42:25,271 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f73de102fbd6447797db31daeaecde9f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/B/f73de102fbd6447797db31daeaecde9f 2024-11-28T03:42:25,273 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/343c062cfd8342a185aff2edca233773 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/343c062cfd8342a185aff2edca233773 2024-11-28T03:42:25,273 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/66f637806069442ebd8b2e309c36cee1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/66f637806069442ebd8b2e309c36cee1 2024-11-28T03:42:25,274 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/bf8775f95fdd4708b7cdf82552cb5edc to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/C/bf8775f95fdd4708b7cdf82552cb5edc 2024-11-28T03:42:25,276 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/recovered.edits/813.seqid to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75/recovered.edits/813.seqid 2024-11-28T03:42:25,277 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/1a2c0de76d7e5c66916b7f6387695a75 2024-11-28T03:42:25,277 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T03:42:25,279 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:25,281 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T03:42:25,283 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T03:42:25,284 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:25,284 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T03:42:25,284 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732765345284"}]},"ts":"9223372036854775807"} 2024-11-28T03:42:25,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T03:42:25,286 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1a2c0de76d7e5c66916b7f6387695a75, NAME => 'TestAcidGuarantees,,1732765317306.1a2c0de76d7e5c66916b7f6387695a75.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T03:42:25,286 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T03:42:25,286 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732765345286"}]},"ts":"9223372036854775807"} 2024-11-28T03:42:25,288 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T03:42:25,288 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:25,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-28T03:42:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-28T03:42:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T03:42:25,367 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:25,379 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 251), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=486 (was 526), ProcessCount=11 (was 11), AvailableMemoryMB=6266 (was 6880) 2024-11-28T03:42:25,388 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=486, ProcessCount=11, AvailableMemoryMB=6266 2024-11-28T03:42:25,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T03:42:25,390 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:42:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:25,392 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:42:25,392 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:25,392 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-28T03:42:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T03:42:25,393 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:42:25,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742199_1375 (size=1061) 2024-11-28T03:42:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T03:42:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T03:42:25,801 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72 2024-11-28T03:42:25,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742200_1376 (size=53) 2024-11-28T03:42:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T03:42:26,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:42:26,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing eb2f0a77050be706d70e7ee7532fcde6, disabling compactions & flushes 2024-11-28T03:42:26,207 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:26,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:26,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. after waiting 0 ms 2024-11-28T03:42:26,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:26,208 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:26,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for eb2f0a77050be706d70e7ee7532fcde6: Waiting for close lock at 1732765346207Disabling compacts and flushes for region at 1732765346207Disabling writes for close at 1732765346208 (+1 ms)Writing region close event to WAL at 1732765346208Closed at 1732765346208 2024-11-28T03:42:26,209 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:42:26,209 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732765346209"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732765346209"}]},"ts":"1732765346209"} 2024-11-28T03:42:26,211 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:42:26,212 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:42:26,212 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765346212"}]},"ts":"1732765346212"} 2024-11-28T03:42:26,213 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T03:42:26,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, ASSIGN}] 2024-11-28T03:42:26,215 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, ASSIGN 2024-11-28T03:42:26,215 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, ASSIGN; state=OFFLINE, location=3a546af21488,34671,1732765249447; forceNewPlan=false, retain=false 2024-11-28T03:42:26,366 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=eb2f0a77050be706d70e7ee7532fcde6, regionState=OPENING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:42:26,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, ASSIGN because future has completed 2024-11-28T03:42:26,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:42:26,522 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:26,522 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:42:26,523 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,523 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:42:26,523 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,523 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,524 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,525 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:42:26,525 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb2f0a77050be706d70e7ee7532fcde6 columnFamilyName A 2024-11-28T03:42:26,525 DEBUG [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:26,526 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(327): Store=eb2f0a77050be706d70e7ee7532fcde6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:42:26,526 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T03:42:26,527 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:42:26,527 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb2f0a77050be706d70e7ee7532fcde6 columnFamilyName B 2024-11-28T03:42:26,527 DEBUG [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:26,527 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(327): Store=eb2f0a77050be706d70e7ee7532fcde6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:42:26,527 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,528 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:42:26,528 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb2f0a77050be706d70e7ee7532fcde6 columnFamilyName C 2024-11-28T03:42:26,528 DEBUG [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:26,529 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(327): Store=eb2f0a77050be706d70e7ee7532fcde6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:42:26,529 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:26,529 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,529 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,530 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,530 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,530 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,531 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:42:26,532 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,534 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:42:26,534 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened eb2f0a77050be706d70e7ee7532fcde6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68740316, jitterRate=0.024310529232025146}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:42:26,534 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:26,535 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for eb2f0a77050be706d70e7ee7532fcde6: Running coprocessor pre-open hook at 1732765346523Writing region info on filesystem at 1732765346523Initializing all the Stores at 1732765346524 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765346524Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765346524Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765346524Cleaning up temporary data from old regions at 1732765346530 (+6 ms)Running coprocessor post-open hooks at 1732765346534 (+4 ms)Region opened successfully at 1732765346535 (+1 ms) 2024-11-28T03:42:26,535 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., pid=99, masterSystemTime=1732765346519 2024-11-28T03:42:26,537 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:26,537 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:26,537 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=eb2f0a77050be706d70e7ee7532fcde6, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:42:26,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:42:26,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-28T03:42:26,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 in 171 msec 2024-11-28T03:42:26,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-28T03:42:26,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, ASSIGN in 327 msec 2024-11-28T03:42:26,542 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:42:26,543 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765346542"}]},"ts":"1732765346542"} 2024-11-28T03:42:26,544 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T03:42:26,544 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:42:26,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-28T03:42:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T03:42:27,546 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:27,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd6c5bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:27,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:27,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:27,549 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@835426e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:27,550 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fece5ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:27,551 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:27,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:27,553 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T03:42:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3a546af21488,38327,1732765248628 2024-11-28T03:42:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@25608cc7 2024-11-28T03:42:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T03:42:27,555 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T03:42:27,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T03:42:27,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:42:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:27,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742201_1377 (size=1097) 2024-11-28T03:42:27,968 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T03:42:27,968 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T03:42:27,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T03:42:27,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, REOPEN/MOVE}] 2024-11-28T03:42:27,973 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, REOPEN/MOVE 2024-11-28T03:42:27,974 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=eb2f0a77050be706d70e7ee7532fcde6, regionState=CLOSING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:42:27,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, REOPEN/MOVE because future has completed 2024-11-28T03:42:27,975 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T03:42:27,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:42:28,128 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,128 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T03:42:28,128 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing eb2f0a77050be706d70e7ee7532fcde6, disabling compactions & flushes 2024-11-28T03:42:28,128 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,128 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,128 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. after waiting 0 ms 2024-11-28T03:42:28,128 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,132 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T03:42:28,132 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,133 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for eb2f0a77050be706d70e7ee7532fcde6: Waiting for close lock at 1732765348128Running coprocessor pre-close hooks at 1732765348128Disabling compacts and flushes for region at 1732765348128Disabling writes for close at 1732765348128Writing region close event to WAL at 1732765348129 (+1 ms)Running coprocessor post-close hooks at 1732765348132 (+3 ms)Closed at 1732765348132 2024-11-28T03:42:28,133 WARN [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: eb2f0a77050be706d70e7ee7532fcde6 to self. 2024-11-28T03:42:28,134 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,135 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=eb2f0a77050be706d70e7ee7532fcde6, regionState=CLOSED 2024-11-28T03:42:28,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:42:28,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-28T03:42:28,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 in 161 msec 2024-11-28T03:42:28,138 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, REOPEN/MOVE; state=CLOSED, location=3a546af21488,34671,1732765249447; forceNewPlan=false, retain=true 2024-11-28T03:42:28,289 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=eb2f0a77050be706d70e7ee7532fcde6, regionState=OPENING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, REOPEN/MOVE because future has completed 2024-11-28T03:42:28,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:42:28,446 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,446 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:42:28,446 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,446 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:42:28,446 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,446 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,447 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,448 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:42:28,448 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb2f0a77050be706d70e7ee7532fcde6 columnFamilyName A 2024-11-28T03:42:28,450 DEBUG [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:28,450 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(327): Store=eb2f0a77050be706d70e7ee7532fcde6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:42:28,450 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,451 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:42:28,451 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb2f0a77050be706d70e7ee7532fcde6 columnFamilyName B 2024-11-28T03:42:28,451 DEBUG [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:28,452 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(327): Store=eb2f0a77050be706d70e7ee7532fcde6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:42:28,452 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,452 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:42:28,452 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb2f0a77050be706d70e7ee7532fcde6 columnFamilyName C 2024-11-28T03:42:28,452 DEBUG [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:28,453 INFO [StoreOpener-eb2f0a77050be706d70e7ee7532fcde6-1 {}] regionserver.HStore(327): Store=eb2f0a77050be706d70e7ee7532fcde6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:42:28,453 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,453 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,453 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,454 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,455 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,455 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,455 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:42:28,456 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,457 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened eb2f0a77050be706d70e7ee7532fcde6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60807950, jitterRate=-0.09389093518257141}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:42:28,457 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,457 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for eb2f0a77050be706d70e7ee7532fcde6: Running coprocessor pre-open hook at 1732765348446Writing region info on filesystem at 1732765348446Initializing all the Stores at 1732765348447 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765348447Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765348447Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765348447Cleaning up temporary data from old regions at 1732765348455 (+8 ms)Running coprocessor post-open hooks at 1732765348457 (+2 ms)Region opened successfully at 1732765348457 2024-11-28T03:42:28,458 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., pid=104, masterSystemTime=1732765348443 2024-11-28T03:42:28,459 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,460 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,460 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=eb2f0a77050be706d70e7ee7532fcde6, regionState=OPEN, openSeqNum=5, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:42:28,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-28T03:42:28,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 in 171 msec 2024-11-28T03:42:28,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-28T03:42:28,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, REOPEN/MOVE in 491 msec 2024-11-28T03:42:28,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-28T03:42:28,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 496 msec 2024-11-28T03:42:28,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 912 msec 2024-11-28T03:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-28T03:42:28,471 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:28,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7baf5f2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,472 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:28,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:28,473 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666e8026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:28,474 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce09d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f32b497, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:28,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:28,476 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@624ff824, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:28,477 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa77dd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695c024f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:28,478 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:28,479 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8ac55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:28,480 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5941912e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549586f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:28,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:28,482 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554f7fe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:28,483 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3811bddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cba9b20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:28,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:28,485 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3966cdc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:28,486 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:28,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53c1c13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55314bb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:28,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:28,488 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cb7f602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:28,489 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28f6b34f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b29348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,490 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:28,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:28,491 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d15ea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:28,492 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df21943, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3b32d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:28,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:28,494 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2beaa187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:28,495 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f0b2c39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b79742b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:28,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:28,497 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@351aa1ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:28,498 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee52c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d85bef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:28,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:28,500 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9adb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:28,501 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76663b98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:28,502 DEBUG [Thread-1702 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:28,502 DEBUG [Thread-1706 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:28,502 DEBUG [Thread-1698 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:28,502 DEBUG [Thread-1700 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:28,502 DEBUG [Thread-1709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:28,502 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:28,503 DEBUG [Thread-1711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-28T03:42:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:28,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:28,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:28,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T03:42:28,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:28,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:28,505 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:28,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:28,506 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:28,506 DEBUG [Thread-1717 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:28,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:28,506 DEBUG [Thread-1715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:28,506 DEBUG [Thread-1713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:42:28,507 DEBUG [Thread-1704 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:28,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:28,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:42:28,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:42:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:42:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:42:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:42:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:42:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:42:28,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:42:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:42:28,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:42:28,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:28,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:28,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:28,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:28,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:28,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281cf45cadab6a4969a05e6079490e6e98_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765348508/Put/seqid=0 2024-11-28T03:42:28,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742202_1378 (size=14594) 2024-11-28T03:42:28,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765348530, max timestamp=1732765348565Num uniques 859; 2024-11-28T03:42:28,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765348530, max timestamp=1732765348565Num uniques 859; 2024-11-28T03:42:28,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:28,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:28,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765348530, max timestamp=1732765348565Num uniques 859; 2024-11-28T03:42:28,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T03:42:28,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765408602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765408602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765408603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765408603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765408608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T03:42:28,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:28,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:28,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:28,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765408726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765408727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765408727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765408727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765408727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T03:42:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T03:42:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765408946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765408946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765408946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765408946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765408946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:28,965 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:28,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T03:42:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:28,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:28,969 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281cf45cadab6a4969a05e6079490e6e98_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281cf45cadab6a4969a05e6079490e6e98_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:28,970 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/cd14a988d1d64c45b71820f5db31bb11, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:28,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/cd14a988d1d64c45b71820f5db31bb11 is 175, key is test_row_0/A:col10/1732765348508/Put/seqid=0 2024-11-28T03:42:28,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742203_1379 (size=39549) 2024-11-28T03:42:28,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/cd14a988d1d64c45b71820f5db31bb11 2024-11-28T03:42:29,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/ced927c2b4e348c5a1de06556dc09e07 is 50, key is test_row_0/B:col10/1732765348508/Put/seqid=0 2024-11-28T03:42:29,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742204_1380 (size=12001) 2024-11-28T03:42:29,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T03:42:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T03:42:29,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765409256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:29,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T03:42:29,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:29,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765409266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765409266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:29,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765409267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:29,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765409267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:29,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/ced927c2b4e348c5a1de06556dc09e07 2024-11-28T03:42:29,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T03:42:29,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:29,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/039dc5e7ca1b4ea9a26301a2323c69fe is 50, key is test_row_0/C:col10/1732765348508/Put/seqid=0 2024-11-28T03:42:29,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742205_1381 (size=12001) 2024-11-28T03:42:29,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T03:42:29,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:29,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T03:42:29,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T03:42:29,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:29,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:29,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765409777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:29,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765409786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765409786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765409786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765409786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:29,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/039dc5e7ca1b4ea9a26301a2323c69fe 2024-11-28T03:42:29,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/cd14a988d1d64c45b71820f5db31bb11 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/cd14a988d1d64c45b71820f5db31bb11 2024-11-28T03:42:29,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/cd14a988d1d64c45b71820f5db31bb11, entries=200, sequenceid=14, filesize=38.6 K 2024-11-28T03:42:29,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/ced927c2b4e348c5a1de06556dc09e07 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/ced927c2b4e348c5a1de06556dc09e07 2024-11-28T03:42:29,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/ced927c2b4e348c5a1de06556dc09e07, entries=150, sequenceid=14, filesize=11.7 K 2024-11-28T03:42:29,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/039dc5e7ca1b4ea9a26301a2323c69fe as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/039dc5e7ca1b4ea9a26301a2323c69fe 2024-11-28T03:42:29,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/039dc5e7ca1b4ea9a26301a2323c69fe, entries=150, sequenceid=14, filesize=11.7 K 2024-11-28T03:42:29,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for eb2f0a77050be706d70e7ee7532fcde6 in 1339ms, sequenceid=14, compaction requested=false 2024-11-28T03:42:29,860 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T03:42:29,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:29,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T03:42:29,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:29,885 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-28T03:42:29,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:29,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:29,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:29,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:29,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:29,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:29,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d18b6f5b3f364ebc981df3e5331b7154_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765348601/Put/seqid=0 2024-11-28T03:42:29,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742206_1382 (size=12154) 2024-11-28T03:42:29,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:29,905 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d18b6f5b3f364ebc981df3e5331b7154_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d18b6f5b3f364ebc981df3e5331b7154_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:29,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/c4dd731ec9a24ac7bbaf17c536c16be4, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:29,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/c4dd731ec9a24ac7bbaf17c536c16be4 is 175, key is test_row_0/A:col10/1732765348601/Put/seqid=0 2024-11-28T03:42:29,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742207_1383 (size=30955) 2024-11-28T03:42:30,151 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:42:30,314 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/c4dd731ec9a24ac7bbaf17c536c16be4 2024-11-28T03:42:30,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/d6574371572b4d958e4e752ce78765a5 is 50, key is test_row_0/B:col10/1732765348601/Put/seqid=0 2024-11-28T03:42:30,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742208_1384 (size=12001) 2024-11-28T03:42:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T03:42:30,730 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/d6574371572b4d958e4e752ce78765a5 2024-11-28T03:42:30,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/e7b77600e7864d46803f5c7fcc5cc484 is 50, key is test_row_0/C:col10/1732765348601/Put/seqid=0 2024-11-28T03:42:30,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742209_1385 (size=12001) 2024-11-28T03:42:30,746 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/e7b77600e7864d46803f5c7fcc5cc484 2024-11-28T03:42:30,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/c4dd731ec9a24ac7bbaf17c536c16be4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/c4dd731ec9a24ac7bbaf17c536c16be4 2024-11-28T03:42:30,757 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/c4dd731ec9a24ac7bbaf17c536c16be4, entries=150, sequenceid=49, filesize=30.2 K 2024-11-28T03:42:30,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/d6574371572b4d958e4e752ce78765a5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/d6574371572b4d958e4e752ce78765a5 2024-11-28T03:42:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,763 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/d6574371572b4d958e4e752ce78765a5, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T03:42:30,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/e7b77600e7864d46803f5c7fcc5cc484 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e7b77600e7864d46803f5c7fcc5cc484 2024-11-28T03:42:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,767 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e7b77600e7864d46803f5c7fcc5cc484, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T03:42:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,768 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for eb2f0a77050be706d70e7ee7532fcde6 in 883ms, sequenceid=49, compaction requested=false 2024-11-28T03:42:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:30,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:30,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-28T03:42:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-28T03:42:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-28T03:42:30,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2640 sec 2024-11-28T03:42:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.2700 sec 2024-11-28T03:42:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:42:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:30,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:30,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:30,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:30,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:30,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284e16843ffec74478be15f4ab253c40d8_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765350926/Put/seqid=0 2024-11-28T03:42:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742210_1386 (size=21918) 2024-11-28T03:42:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,970 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,975 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284e16843ffec74478be15f4ab253c40d8_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284e16843ffec74478be15f4ab253c40d8_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,976 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/6f6b4445a65e440199d1c726037954d5, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/6f6b4445a65e440199d1c726037954d5 is 175, key is test_row_0/A:col10/1732765350926/Put/seqid=0 2024-11-28T03:42:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742211_1387 (size=65323) 2024-11-28T03:42:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:31,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732765350988, max timestamp=1732765351017Num uniques 859; 2024-11-28T03:42:31,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T03:42:31,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732765350988, max timestamp=1732765351017Num uniques 859; 2024-11-28T03:42:31,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T03:42:31,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=183, cellSize=8378, totalHeapSize=19272, min timestamp=1732765350990, max timestamp=1732765351017Num uniques 859; 2024-11-28T03:42:31,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8378, suffix heap size=117064,new segment heap size=19272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=183 2024-11-28T03:42:31,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765411055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765411057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765411058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765411059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765411060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765411176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765411177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765411177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765411177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765411186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/6f6b4445a65e440199d1c726037954d5 2024-11-28T03:42:31,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/4153e35401c3453e8fe28dd82c145b5a is 50, key is test_row_0/B:col10/1732765350926/Put/seqid=0 2024-11-28T03:42:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765411396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765411396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765411397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765411397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765411407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742212_1388 (size=12001) 2024-11-28T03:42:31,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765411716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765411716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765411716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765411716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765411716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:31,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/4153e35401c3453e8fe28dd82c145b5a 2024-11-28T03:42:31,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/5f02c1a1d65042e1ad02748c6860133c is 50, key is test_row_0/C:col10/1732765350926/Put/seqid=0 2024-11-28T03:42:31,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742213_1389 (size=12001) 2024-11-28T03:42:32,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/5f02c1a1d65042e1ad02748c6860133c 2024-11-28T03:42:32,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765412227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:32,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/6f6b4445a65e440199d1c726037954d5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/6f6b4445a65e440199d1c726037954d5 2024-11-28T03:42:32,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/6f6b4445a65e440199d1c726037954d5, entries=350, sequenceid=59, filesize=63.8 K 2024-11-28T03:42:32,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/4153e35401c3453e8fe28dd82c145b5a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4153e35401c3453e8fe28dd82c145b5a 2024-11-28T03:42:32,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765412237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:32,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765412237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:32,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765412237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765412237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:32,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4153e35401c3453e8fe28dd82c145b5a, entries=150, sequenceid=59, filesize=11.7 K 2024-11-28T03:42:32,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/5f02c1a1d65042e1ad02748c6860133c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/5f02c1a1d65042e1ad02748c6860133c 2024-11-28T03:42:32,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/5f02c1a1d65042e1ad02748c6860133c, entries=150, sequenceid=59, filesize=11.7 K 2024-11-28T03:42:32,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=135.26 KB/138504 for eb2f0a77050be706d70e7ee7532fcde6 in 1306ms, sequenceid=59, compaction requested=true 2024-11-28T03:42:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:32,251 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:32,251 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:32,252 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:32,252 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:32,252 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/A is initiating minor compaction (all files) 2024-11-28T03:42:32,252 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/B is initiating minor compaction (all files) 2024-11-28T03:42:32,252 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/A in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:32,252 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/B in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:32,252 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/ced927c2b4e348c5a1de06556dc09e07, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/d6574371572b4d958e4e752ce78765a5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4153e35401c3453e8fe28dd82c145b5a] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=35.2 K 2024-11-28T03:42:32,252 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/cd14a988d1d64c45b71820f5db31bb11, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/c4dd731ec9a24ac7bbaf17c536c16be4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/6f6b4445a65e440199d1c726037954d5] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=132.6 K 2024-11-28T03:42:32,252 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:32,252 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/cd14a988d1d64c45b71820f5db31bb11, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/c4dd731ec9a24ac7bbaf17c536c16be4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/6f6b4445a65e440199d1c726037954d5] 2024-11-28T03:42:32,252 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ced927c2b4e348c5a1de06556dc09e07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732765348508 2024-11-28T03:42:32,253 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d6574371572b4d958e4e752ce78765a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732765348601 2024-11-28T03:42:32,253 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd14a988d1d64c45b71820f5db31bb11, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732765348508 2024-11-28T03:42:32,253 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4dd731ec9a24ac7bbaf17c536c16be4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732765348601 2024-11-28T03:42:32,253 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4153e35401c3453e8fe28dd82c145b5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732765350926 2024-11-28T03:42:32,253 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f6b4445a65e440199d1c726037954d5, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732765350920 2024-11-28T03:42:32,261 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:32,261 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/584f28947c1145b299d0e48073f5bf39 is 50, key is test_row_0/B:col10/1732765350926/Put/seqid=0 2024-11-28T03:42:32,264 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:32,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742214_1390 (size=12104) 2024-11-28T03:42:32,276 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128f151114afc6d4692ac35453cdb3e7be9_eb2f0a77050be706d70e7ee7532fcde6 store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:32,279 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128f151114afc6d4692ac35453cdb3e7be9_eb2f0a77050be706d70e7ee7532fcde6, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:32,279 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f151114afc6d4692ac35453cdb3e7be9_eb2f0a77050be706d70e7ee7532fcde6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:32,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742215_1391 (size=4469) 2024-11-28T03:42:32,674 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/584f28947c1145b299d0e48073f5bf39 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/584f28947c1145b299d0e48073f5bf39 2024-11-28T03:42:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T03:42:32,676 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:32,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-28T03:42:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T03:42:32,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:32,680 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:32,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:32,681 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/B of eb2f0a77050be706d70e7ee7532fcde6 into 584f28947c1145b299d0e48073f5bf39(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:32,681 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:32,681 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/B, priority=13, startTime=1732765352251; duration=0sec 2024-11-28T03:42:32,681 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:32,681 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:B 2024-11-28T03:42:32,681 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:32,682 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:32,682 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/C is initiating minor compaction (all files) 2024-11-28T03:42:32,682 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/C in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:32,682 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/039dc5e7ca1b4ea9a26301a2323c69fe, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e7b77600e7864d46803f5c7fcc5cc484, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/5f02c1a1d65042e1ad02748c6860133c] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=35.2 K 2024-11-28T03:42:32,682 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 039dc5e7ca1b4ea9a26301a2323c69fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732765348508 2024-11-28T03:42:32,683 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e7b77600e7864d46803f5c7fcc5cc484, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732765348601 2024-11-28T03:42:32,683 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f02c1a1d65042e1ad02748c6860133c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732765350926 2024-11-28T03:42:32,690 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:32,690 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:32,691 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/d11cd6c434814444ac404d3ce96ba46c is 50, key is test_row_0/C:col10/1732765350926/Put/seqid=0 2024-11-28T03:42:32,691 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/3a047cce000749f7a44c0f98824c53c4 is 175, key is test_row_0/A:col10/1732765350926/Put/seqid=0 2024-11-28T03:42:32,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742216_1392 (size=12104) 2024-11-28T03:42:32,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742217_1393 (size=31058) 2024-11-28T03:42:32,700 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/d11cd6c434814444ac404d3ce96ba46c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d11cd6c434814444ac404d3ce96ba46c 2024-11-28T03:42:32,704 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/C of eb2f0a77050be706d70e7ee7532fcde6 into d11cd6c434814444ac404d3ce96ba46c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:32,704 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:32,704 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/C, priority=13, startTime=1732765352251; duration=0sec 2024-11-28T03:42:32,704 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:32,704 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:C 2024-11-28T03:42:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T03:42:32,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T03:42:32,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:32,833 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=135.26 KB heapSize=378.05 KB 2024-11-28T03:42:32,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:32,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:32,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:32,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:32,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:32,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:32,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284c69ae651d5d4471918bd5ebb0e80f7a_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765351057/Put/seqid=0 2024-11-28T03:42:32,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742218_1394 (size=12154) 2024-11-28T03:42:32,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,854 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284c69ae651d5d4471918bd5ebb0e80f7a_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284c69ae651d5d4471918bd5ebb0e80f7a_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:32,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/3dcef47bf4c84b93a873e031615ac585, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:32,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/3dcef47bf4c84b93a873e031615ac585 is 175, key is test_row_0/A:col10/1732765351057/Put/seqid=0 2024-11-28T03:42:32,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742219_1395 (size=30955) 2024-11-28T03:42:32,862 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/3dcef47bf4c84b93a873e031615ac585 2024-11-28T03:42:32,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/5f88b3deff004e779183efd0e4945d1c is 50, key is test_row_0/B:col10/1732765351057/Put/seqid=0 2024-11-28T03:42:32,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742220_1396 (size=12001) 2024-11-28T03:42:32,885 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/5f88b3deff004e779183efd0e4945d1c 2024-11-28T03:42:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/8269f9eeb9bc49969a2b260d5e514d35 is 50, key is test_row_0/C:col10/1732765351057/Put/seqid=0 2024-11-28T03:42:32,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742221_1397 (size=12001) 2024-11-28T03:42:32,902 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.33 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/8269f9eeb9bc49969a2b260d5e514d35 2024-11-28T03:42:32,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/3dcef47bf4c84b93a873e031615ac585 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3dcef47bf4c84b93a873e031615ac585 2024-11-28T03:42:32,910 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3dcef47bf4c84b93a873e031615ac585, entries=150, sequenceid=96, filesize=30.2 K 2024-11-28T03:42:32,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/5f88b3deff004e779183efd0e4945d1c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/5f88b3deff004e779183efd0e4945d1c 2024-11-28T03:42:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,915 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/5f88b3deff004e779183efd0e4945d1c, entries=150, sequenceid=96, filesize=11.7 K 2024-11-28T03:42:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/8269f9eeb9bc49969a2b260d5e514d35 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/8269f9eeb9bc49969a2b260d5e514d35 2024-11-28T03:42:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,921 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/8269f9eeb9bc49969a2b260d5e514d35, entries=150, sequenceid=96, filesize=11.7 K 2024-11-28T03:42:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,922 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~135.26 KB/138504, heapSize ~378 KB/387072, currentSize=0 B/0 for eb2f0a77050be706d70e7ee7532fcde6 in 89ms, sequenceid=96, compaction requested=false 2024-11-28T03:42:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:32,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:32,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-28T03:42:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-28T03:42:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-28T03:42:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 243 msec 2024-11-28T03:42:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 249 msec 2024-11-28T03:42:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T03:42:32,997 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-28T03:42:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T03:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,000 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,001 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:33,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,102 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/3a047cce000749f7a44c0f98824c53c4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3a047cce000749f7a44c0f98824c53c4 2024-11-28T03:42:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T03:42:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,112 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/A of eb2f0a77050be706d70e7ee7532fcde6 into 3a047cce000749f7a44c0f98824c53c4(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:33,112 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:33,112 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/A, priority=13, startTime=1732765352251; duration=0sec 2024-11-28T03:42:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,112 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:33,112 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:A 2024-11-28T03:42:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T03:42:33,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:33,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:33,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-28T03:42:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-28T03:42:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-28T03:42:33,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 157 msec 2024-11-28T03:42:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 164 msec 2024-11-28T03:42:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T03:42:33,317 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:33,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,318 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:33,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-28T03:42:33,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T03:42:33,320 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:33,321 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:33,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:33,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T03:42:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T03:42:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,484 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T03:42:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:33,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f6ab0761e15d44999fbde0f0f6ce2588_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765353423/Put/seqid=0 2024-11-28T03:42:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742222_1398 (size=17034) 2024-11-28T03:42:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:33,584 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f6ab0761e15d44999fbde0f0f6ce2588_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f6ab0761e15d44999fbde0f0f6ce2588_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:33,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/fd5307964d3449baa2efc7c38c421882, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:33,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/fd5307964d3449baa2efc7c38c421882 is 175, key is test_row_0/A:col10/1732765353423/Put/seqid=0 2024-11-28T03:42:33,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742223_1399 (size=48135) 2024-11-28T03:42:33,625 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/fd5307964d3449baa2efc7c38c421882 2024-11-28T03:42:33,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/347167cdfbd94caa9d37dde2c98c9edd is 50, key is test_row_0/B:col10/1732765353423/Put/seqid=0 2024-11-28T03:42:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T03:42:33,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765353588, max timestamp=1732765353647Num uniques 859; 2024-11-28T03:42:33,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:33,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765353588, max timestamp=1732765353647Num uniques 859; 2024-11-28T03:42:33,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:33,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732765353588, max timestamp=1732765353647Num uniques 859; 2024-11-28T03:42:33,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T03:42:33,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742224_1400 (size=9657) 2024-11-28T03:42:33,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765413715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:33,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765413720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:33,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:33,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765413727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:33,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765413728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765413730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:33,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765413847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:33,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765413847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:33,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765413857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765413858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:33,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765413858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T03:42:34,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765414066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765414076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,084 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/347167cdfbd94caa9d37dde2c98c9edd 2024-11-28T03:42:34,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/506c57cd658a481c9207eb3dc357de3c is 50, key is test_row_0/C:col10/1732765353423/Put/seqid=0 2024-11-28T03:42:34,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765414087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765414087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765414087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742225_1401 (size=9657) 2024-11-28T03:42:34,122 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/506c57cd658a481c9207eb3dc357de3c 2024-11-28T03:42:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/fd5307964d3449baa2efc7c38c421882 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/fd5307964d3449baa2efc7c38c421882 2024-11-28T03:42:34,140 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/fd5307964d3449baa2efc7c38c421882, entries=250, sequenceid=106, filesize=47.0 K 2024-11-28T03:42:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/347167cdfbd94caa9d37dde2c98c9edd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/347167cdfbd94caa9d37dde2c98c9edd 2024-11-28T03:42:34,145 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/347167cdfbd94caa9d37dde2c98c9edd, entries=100, sequenceid=106, filesize=9.4 K 2024-11-28T03:42:34,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/506c57cd658a481c9207eb3dc357de3c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/506c57cd658a481c9207eb3dc357de3c 2024-11-28T03:42:34,153 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/506c57cd658a481c9207eb3dc357de3c, entries=100, sequenceid=106, filesize=9.4 K 2024-11-28T03:42:34,154 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for eb2f0a77050be706d70e7ee7532fcde6 in 670ms, sequenceid=106, compaction requested=true 2024-11-28T03:42:34,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:34,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:34,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-28T03:42:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-28T03:42:34,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-28T03:42:34,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 835 msec 2024-11-28T03:42:34,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 841 msec 2024-11-28T03:42:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:34,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-11-28T03:42:34,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765353724, max timestamp=1732765354391Num uniques 1118; 2024-11-28T03:42:34,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765353724, max timestamp=1732765354391Num uniques 1118; 2024-11-28T03:42:34,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765353724, max timestamp=1732765354391Num uniques 1118; 2024-11-28T03:42:34,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-28T03:42:34,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-28T03:42:34,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-28T03:42:34,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:34,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:34,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:34,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:34,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:34,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:34,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c2ebf0e5dfbf4b17bb021f51b3b5a136_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765354393/Put/seqid=0 2024-11-28T03:42:34,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742226_1402 (size=17284) 2024-11-28T03:42:34,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732765354422, max timestamp=1732765354444Num uniques 859; 2024-11-28T03:42:34,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732765354422, max timestamp=1732765354444Num uniques 859; 2024-11-28T03:42:34,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-28T03:42:34,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732765354422, max timestamp=1732765354444Num uniques 859; 2024-11-28T03:42:34,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-28T03:42:34,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-28T03:42:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T03:42:34,456 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:34,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-28T03:42:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T03:42:34,459 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:34,459 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:34,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:34,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765414488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765414492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765414493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765414495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765414497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T03:42:34,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765414606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:34,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:34,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:34,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:34,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765414616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765414616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765414616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765414617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:34,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:34,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:34,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:34,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:34,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T03:42:34,812 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:34,816 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c2ebf0e5dfbf4b17bb021f51b3b5a136_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c2ebf0e5dfbf4b17bb021f51b3b5a136_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:34,818 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/42f4f9dd60e24f35b69671fd972465c2, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:34,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/42f4f9dd60e24f35b69671fd972465c2 is 175, key is test_row_0/A:col10/1732765354393/Put/seqid=0 2024-11-28T03:42:34,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742227_1403 (size=48389) 2024-11-28T03:42:34,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765414816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765414836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765414836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765414837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765414837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:34,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:34,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:34,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:34,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:34,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:34,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:35,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:35,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T03:42:35,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765415126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:35,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765415156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:35,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765415157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:35,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765415157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:35,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765415157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:35,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/42f4f9dd60e24f35b69671fd972465c2 2024-11-28T03:42:35,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:35,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:35,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/92cb87602b2c4060977086962dce9b7d is 50, key is test_row_0/B:col10/1732765354393/Put/seqid=0 2024-11-28T03:42:35,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742228_1404 (size=12151) 2024-11-28T03:42:35,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/92cb87602b2c4060977086962dce9b7d 2024-11-28T03:42:35,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/519f3fa6fa9e49ff8a111ae94d318626 is 50, key is test_row_0/C:col10/1732765354393/Put/seqid=0 2024-11-28T03:42:35,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742229_1405 (size=12151) 2024-11-28T03:42:35,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:35,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:35,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:35,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:35,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T03:42:35,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/519f3fa6fa9e49ff8a111ae94d318626 2024-11-28T03:42:35,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/42f4f9dd60e24f35b69671fd972465c2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/42f4f9dd60e24f35b69671fd972465c2 2024-11-28T03:42:35,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765415646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:35,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:35,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/42f4f9dd60e24f35b69671fd972465c2, entries=250, sequenceid=145, filesize=47.3 K 2024-11-28T03:42:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/92cb87602b2c4060977086962dce9b7d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/92cb87602b2c4060977086962dce9b7d 2024-11-28T03:42:35,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/92cb87602b2c4060977086962dce9b7d, entries=150, sequenceid=145, filesize=11.9 K 2024-11-28T03:42:35,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/519f3fa6fa9e49ff8a111ae94d318626 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/519f3fa6fa9e49ff8a111ae94d318626 2024-11-28T03:42:35,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/519f3fa6fa9e49ff8a111ae94d318626, entries=150, sequenceid=145, filesize=11.9 K 2024-11-28T03:42:35,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=161.02 KB/164880 for eb2f0a77050be706d70e7ee7532fcde6 in 1281ms, sequenceid=145, compaction requested=true 2024-11-28T03:42:35,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:35,675 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:42:35,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:35,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:35,676 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:42:35,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:35,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:35,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:35,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:35,678 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158537 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:42:35,678 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/A is initiating minor compaction (all files) 2024-11-28T03:42:35,678 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/A in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,678 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3a047cce000749f7a44c0f98824c53c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3dcef47bf4c84b93a873e031615ac585, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/fd5307964d3449baa2efc7c38c421882, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/42f4f9dd60e24f35b69671fd972465c2] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=154.8 K 2024-11-28T03:42:35,678 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,679 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3a047cce000749f7a44c0f98824c53c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3dcef47bf4c84b93a873e031615ac585, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/fd5307964d3449baa2efc7c38c421882, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/42f4f9dd60e24f35b69671fd972465c2] 2024-11-28T03:42:35,679 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a047cce000749f7a44c0f98824c53c4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732765350926 2024-11-28T03:42:35,679 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:42:35,679 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/B is initiating minor compaction (all files) 2024-11-28T03:42:35,679 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/B in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,679 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/584f28947c1145b299d0e48073f5bf39, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/5f88b3deff004e779183efd0e4945d1c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/347167cdfbd94caa9d37dde2c98c9edd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/92cb87602b2c4060977086962dce9b7d] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=44.8 K 2024-11-28T03:42:35,680 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dcef47bf4c84b93a873e031615ac585, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732765351053 2024-11-28T03:42:35,680 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 584f28947c1145b299d0e48073f5bf39, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732765350926 2024-11-28T03:42:35,680 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd5307964d3449baa2efc7c38c421882, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732765353404 2024-11-28T03:42:35,680 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f88b3deff004e779183efd0e4945d1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732765351053 2024-11-28T03:42:35,680 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42f4f9dd60e24f35b69671fd972465c2, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732765353724 2024-11-28T03:42:35,681 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 347167cdfbd94caa9d37dde2c98c9edd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732765353423 2024-11-28T03:42:35,681 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 92cb87602b2c4060977086962dce9b7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732765353724 2024-11-28T03:42:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:35,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=167.72 KB heapSize=461.16 KB 2024-11-28T03:42:35,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:35,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:35,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732765354490, max timestamp=1732765355683Num uniques 1268; 2024-11-28T03:42:35,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732765354490, max timestamp=1732765355683Num uniques 1268; 2024-11-28T03:42:35,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=58068, new segment data size=9978, suffix heap size=159840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1268, new segment cells count=218 2024-11-28T03:42:35,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=58068, new segment data size=9978, suffix heap size=159840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1268, new segment cells count=218 2024-11-28T03:42:35,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732765354490, max timestamp=1732765355683Num uniques 1268; 2024-11-28T03:42:35,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=58068, new segment data size=9978, suffix heap size=159840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1268, new segment cells count=218 2024-11-28T03:42:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:35,696 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#B#compaction#351 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:35,696 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:35,697 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/e5b2e27d4c32407fa59051410e67c31d is 50, key is test_row_0/B:col10/1732765354393/Put/seqid=0 2024-11-28T03:42:35,698 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112813e84b65fdb04ba286d4657d40569743_eb2f0a77050be706d70e7ee7532fcde6 store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:35,701 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112813e84b65fdb04ba286d4657d40569743_eb2f0a77050be706d70e7ee7532fcde6, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:35,701 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112813e84b65fdb04ba286d4657d40569743_eb2f0a77050be706d70e7ee7532fcde6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:35,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112808137823404747e9958dbf2c5dffb814_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765355683/Put/seqid=0 2024-11-28T03:42:35,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742231_1407 (size=4469) 2024-11-28T03:42:35,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742230_1406 (size=12391) 2024-11-28T03:42:35,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742232_1408 (size=12304) 2024-11-28T03:42:35,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732765355726, max timestamp=1732765355747Num uniques 859; 2024-11-28T03:42:35,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T03:42:35,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732765355726, max timestamp=1732765355747Num uniques 859; 2024-11-28T03:42:35,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T03:42:35,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732765355726, max timestamp=1732765355747Num uniques 859; 2024-11-28T03:42:35,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T03:42:35,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732765355787, max timestamp=1732765355819Num uniques 1168; 2024-11-28T03:42:35,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-11-28T03:42:35,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732765355787, max timestamp=1732765355819Num uniques 1168; 2024-11-28T03:42:35,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732765355787, max timestamp=1732765355819Num uniques 1168; 2024-11-28T03:42:35,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-11-28T03:42:35,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-11-28T03:42:35,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:35,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:35,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732765355850, max timestamp=1732765355887Num uniques 1177; 2024-11-28T03:42:35,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-28T03:42:35,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732765355850, max timestamp=1732765355887Num uniques 1177; 2024-11-28T03:42:35,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-28T03:42:35,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732765355850, max timestamp=1732765355887Num uniques 1177; 2024-11-28T03:42:35,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-28T03:42:35,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732765355932, max timestamp=1732765355949Num uniques 1036; 2024-11-28T03:42:35,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-28T03:42:35,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732765355932, max timestamp=1732765355949Num uniques 1036; 2024-11-28T03:42:35,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-28T03:42:35,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732765355932, max timestamp=1732765355949Num uniques 1036; 2024-11-28T03:42:35,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-28T03:42:35,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:35,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:35,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:35,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1732765355977, max timestamp=1732765356016Num uniques 1145; 2024-11-28T03:42:36,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-11-28T03:42:36,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1732765355977, max timestamp=1732765356016Num uniques 1145; 2024-11-28T03:42:36,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-11-28T03:42:36,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1732765355977, max timestamp=1732765356016Num uniques 1145; 2024-11-28T03:42:36,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-11-28T03:42:36,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765416086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765416090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765416091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765416092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,108 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#A#compaction#352 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:36,108 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/ec5abbca4e224ab58a4060279649e2d0 is 175, key is test_row_0/A:col10/1732765354393/Put/seqid=0 2024-11-28T03:42:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742233_1409 (size=31345) 2024-11-28T03:42:36,114 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:36,114 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/e5b2e27d4c32407fa59051410e67c31d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e5b2e27d4c32407fa59051410e67c31d 2024-11-28T03:42:36,117 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112808137823404747e9958dbf2c5dffb814_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112808137823404747e9958dbf2c5dffb814_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:36,118 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/efb6f11b37b841cdb71831d3ffcf984e, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:36,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/efb6f11b37b841cdb71831d3ffcf984e is 175, key is test_row_0/A:col10/1732765355683/Put/seqid=0 2024-11-28T03:42:36,119 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/B of eb2f0a77050be706d70e7ee7532fcde6 into e5b2e27d4c32407fa59051410e67c31d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:36,119 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:36,119 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/B, priority=12, startTime=1732765355676; duration=0sec 2024-11-28T03:42:36,119 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:36,119 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:B 2024-11-28T03:42:36,119 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:42:36,120 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:42:36,120 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/C is initiating minor compaction (all files) 2024-11-28T03:42:36,120 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/C in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,120 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d11cd6c434814444ac404d3ce96ba46c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/8269f9eeb9bc49969a2b260d5e514d35, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/506c57cd658a481c9207eb3dc357de3c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/519f3fa6fa9e49ff8a111ae94d318626] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=44.8 K 2024-11-28T03:42:36,121 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d11cd6c434814444ac404d3ce96ba46c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732765350926 2024-11-28T03:42:36,121 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8269f9eeb9bc49969a2b260d5e514d35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732765351053 2024-11-28T03:42:36,121 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 506c57cd658a481c9207eb3dc357de3c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732765353423 2024-11-28T03:42:36,122 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 519f3fa6fa9e49ff8a111ae94d318626, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732765353724 2024-11-28T03:42:36,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742234_1410 (size=31105) 2024-11-28T03:42:36,122 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/efb6f11b37b841cdb71831d3ffcf984e 2024-11-28T03:42:36,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/740d473820d74bbd94fd8fbaafde276b is 50, key is test_row_0/B:col10/1732765355683/Put/seqid=0 2024-11-28T03:42:36,139 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#C#compaction#355 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:36,140 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/36894d639edc4c869675eb859397eb1b is 50, key is test_row_0/C:col10/1732765354393/Put/seqid=0 2024-11-28T03:42:36,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742235_1411 (size=12151) 2024-11-28T03:42:36,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/740d473820d74bbd94fd8fbaafde276b 2024-11-28T03:42:36,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:36,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:36,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742236_1412 (size=12391) 2024-11-28T03:42:36,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/7e29ea7fcd13449ebab4084be8c89dff is 50, key is test_row_0/C:col10/1732765355683/Put/seqid=0 2024-11-28T03:42:36,161 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/36894d639edc4c869675eb859397eb1b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/36894d639edc4c869675eb859397eb1b 2024-11-28T03:42:36,166 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/C of eb2f0a77050be706d70e7ee7532fcde6 into 36894d639edc4c869675eb859397eb1b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:36,166 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:36,166 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/C, priority=12, startTime=1732765355676; duration=0sec 2024-11-28T03:42:36,166 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:36,166 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:C 2024-11-28T03:42:36,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742237_1413 (size=12151) 2024-11-28T03:42:36,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/7e29ea7fcd13449ebab4084be8c89dff 2024-11-28T03:42:36,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/efb6f11b37b841cdb71831d3ffcf984e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/efb6f11b37b841cdb71831d3ffcf984e 2024-11-28T03:42:36,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/efb6f11b37b841cdb71831d3ffcf984e, entries=150, sequenceid=183, filesize=30.4 K 2024-11-28T03:42:36,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/740d473820d74bbd94fd8fbaafde276b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/740d473820d74bbd94fd8fbaafde276b 2024-11-28T03:42:36,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/740d473820d74bbd94fd8fbaafde276b, entries=150, sequenceid=183, filesize=11.9 K 2024-11-28T03:42:36,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/7e29ea7fcd13449ebab4084be8c89dff as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7e29ea7fcd13449ebab4084be8c89dff 2024-11-28T03:42:36,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7e29ea7fcd13449ebab4084be8c89dff, entries=150, sequenceid=183, filesize=11.9 K 2024-11-28T03:42:36,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=161.02 KB/164880 for eb2f0a77050be706d70e7ee7532fcde6 in 506ms, sequenceid=183, compaction requested=false 2024-11-28T03:42:36,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:36,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=167.72 KB heapSize=462.47 KB 2024-11-28T03:42:36,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:36,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:36,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:36,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:36,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:36,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:36,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128188da920d5e64f11b913c1e413c77f51_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765356084/Put/seqid=0 2024-11-28T03:42:36,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765416216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765416218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742238_1414 (size=14994) 2024-11-28T03:42:36,228 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:36,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,232 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128188da920d5e64f11b913c1e413c77f51_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128188da920d5e64f11b913c1e413c77f51_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765416222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,233 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/566077488d6f4952b797b1a7eaa6431d, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:36,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/566077488d6f4952b797b1a7eaa6431d is 175, key is test_row_0/A:col10/1732765356084/Put/seqid=0 2024-11-28T03:42:36,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765416224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742239_1415 (size=39949) 2024-11-28T03:42:36,240 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/566077488d6f4952b797b1a7eaa6431d 2024-11-28T03:42:36,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/e7a3a64cb9f5473a804a24901dc75075 is 50, key is test_row_0/B:col10/1732765356084/Put/seqid=0 2024-11-28T03:42:36,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742240_1416 (size=12301) 2024-11-28T03:42:36,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/e7a3a64cb9f5473a804a24901dc75075 2024-11-28T03:42:36,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/81e678bb47374f32a2a0aa1b524562bf is 50, key is test_row_0/C:col10/1732765356084/Put/seqid=0 2024-11-28T03:42:36,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742241_1417 (size=12301) 2024-11-28T03:42:36,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:36,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:36,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765416326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765416336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765416346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765416426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:36,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:36,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,519 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/ec5abbca4e224ab58a4060279649e2d0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec5abbca4e224ab58a4060279649e2d0 2024-11-28T03:42:36,524 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/A of eb2f0a77050be706d70e7ee7532fcde6 into ec5abbca4e224ab58a4060279649e2d0(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:36,524 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:36,524 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/A, priority=12, startTime=1732765355675; duration=0sec 2024-11-28T03:42:36,524 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:36,524 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:A 2024-11-28T03:42:36,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765416546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765416546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765416567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T03:42:36,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:36,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:36,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765416677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,679 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in eb2f0a77050be706d70e7ee7532fcde6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/81e678bb47374f32a2a0aa1b524562bf 2024-11-28T03:42:36,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/566077488d6f4952b797b1a7eaa6431d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/566077488d6f4952b797b1a7eaa6431d 2024-11-28T03:42:36,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/566077488d6f4952b797b1a7eaa6431d, entries=200, sequenceid=291, filesize=39.0 K 2024-11-28T03:42:36,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/e7a3a64cb9f5473a804a24901dc75075 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e7a3a64cb9f5473a804a24901dc75075 2024-11-28T03:42:36,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e7a3a64cb9f5473a804a24901dc75075, entries=150, sequenceid=291, filesize=12.0 K 2024-11-28T03:42:36,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/81e678bb47374f32a2a0aa1b524562bf as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/81e678bb47374f32a2a0aa1b524562bf 2024-11-28T03:42:36,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/81e678bb47374f32a2a0aa1b524562bf, entries=150, sequenceid=291, filesize=12.0 K 2024-11-28T03:42:36,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~462.42 KB/473520, currentSize=20.13 KB/20610 for eb2f0a77050be706d70e7ee7532fcde6 in 510ms, sequenceid=291, compaction requested=true 2024-11-28T03:42:36,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:36,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:36,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:36,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:36,716 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:36,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:36,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:36,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T03:42:36,717 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:36,718 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:36,718 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/A is initiating minor compaction (all files) 2024-11-28T03:42:36,718 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:36,718 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/A in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,718 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/B is initiating minor compaction (all files) 2024-11-28T03:42:36,718 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/B in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,718 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e5b2e27d4c32407fa59051410e67c31d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/740d473820d74bbd94fd8fbaafde276b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e7a3a64cb9f5473a804a24901dc75075] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=36.0 K 2024-11-28T03:42:36,718 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec5abbca4e224ab58a4060279649e2d0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/efb6f11b37b841cdb71831d3ffcf984e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/566077488d6f4952b797b1a7eaa6431d] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=100.0 K 2024-11-28T03:42:36,718 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,718 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec5abbca4e224ab58a4060279649e2d0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/efb6f11b37b841cdb71831d3ffcf984e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/566077488d6f4952b797b1a7eaa6431d] 2024-11-28T03:42:36,719 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b2e27d4c32407fa59051410e67c31d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732765353724 2024-11-28T03:42:36,719 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec5abbca4e224ab58a4060279649e2d0, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732765353724 2024-11-28T03:42:36,719 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 740d473820d74bbd94fd8fbaafde276b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732765354490 2024-11-28T03:42:36,719 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb6f11b37b841cdb71831d3ffcf984e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732765354490 2024-11-28T03:42:36,720 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a3a64cb9f5473a804a24901dc75075, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765356084 2024-11-28T03:42:36,720 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 566077488d6f4952b797b1a7eaa6431d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765356084 2024-11-28T03:42:36,729 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:36,730 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/b5823df254ca460ca4a9412b9d7ed74a is 50, key is test_row_0/B:col10/1732765356084/Put/seqid=0 2024-11-28T03:42:36,733 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:36,744 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128be0c460be3324371ab4e43ea73dcf540_eb2f0a77050be706d70e7ee7532fcde6 store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:36,745 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128be0c460be3324371ab4e43ea73dcf540_eb2f0a77050be706d70e7ee7532fcde6, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:36,745 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128be0c460be3324371ab4e43ea73dcf540_eb2f0a77050be706d70e7ee7532fcde6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:36,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742243_1419 (size=4469) 2024-11-28T03:42:36,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742242_1418 (size=12643) 2024-11-28T03:42:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:36,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:42:36,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:36,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:36,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:36,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:36,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:36,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:36,761 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/b5823df254ca460ca4a9412b9d7ed74a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b5823df254ca460ca4a9412b9d7ed74a 2024-11-28T03:42:36,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:36,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:36,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112882ceda026bf9486aa5497f133641e802_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765356751/Put/seqid=0 2024-11-28T03:42:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,774 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/B of eb2f0a77050be706d70e7ee7532fcde6 into b5823df254ca460ca4a9412b9d7ed74a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:36,774 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:36,774 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/B, priority=13, startTime=1732765356716; duration=0sec 2024-11-28T03:42:36,775 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:36,775 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:B 2024-11-28T03:42:36,775 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:36,776 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:36,776 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/C is initiating minor compaction (all files) 2024-11-28T03:42:36,776 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/C in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,777 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/36894d639edc4c869675eb859397eb1b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7e29ea7fcd13449ebab4084be8c89dff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/81e678bb47374f32a2a0aa1b524562bf] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=36.0 K 2024-11-28T03:42:36,777 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 36894d639edc4c869675eb859397eb1b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732765353724 2024-11-28T03:42:36,777 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e29ea7fcd13449ebab4084be8c89dff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732765354490 2024-11-28T03:42:36,778 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e678bb47374f32a2a0aa1b524562bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765356084 2024-11-28T03:42:36,786 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:36,786 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/b3dc511c2cb54750aa374ebcf01b9b24 is 50, key is test_row_0/C:col10/1732765356084/Put/seqid=0 2024-11-28T03:42:36,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742244_1420 (size=14994) 2024-11-28T03:42:36,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742245_1421 (size=12643) 2024-11-28T03:42:36,792 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:36,797 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112882ceda026bf9486aa5497f133641e802_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112882ceda026bf9486aa5497f133641e802_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:36,800 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/874edc99185b4468bd2ff1bf1e1d107f, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:36,801 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/b3dc511c2cb54750aa374ebcf01b9b24 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/b3dc511c2cb54750aa374ebcf01b9b24 2024-11-28T03:42:36,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/874edc99185b4468bd2ff1bf1e1d107f is 175, key is test_row_0/A:col10/1732765356751/Put/seqid=0 2024-11-28T03:42:36,806 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/C of eb2f0a77050be706d70e7ee7532fcde6 into b3dc511c2cb54750aa374ebcf01b9b24(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:36,806 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:36,806 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/C, priority=13, startTime=1732765356716; duration=0sec 2024-11-28T03:42:36,806 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:36,806 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:C 2024-11-28T03:42:36,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742246_1422 (size=39949) 2024-11-28T03:42:36,811 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/874edc99185b4468bd2ff1bf1e1d107f 2024-11-28T03:42:36,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/f3452f58f0534aff9db8291718e4e563 is 50, key is test_row_0/B:col10/1732765356751/Put/seqid=0 2024-11-28T03:42:36,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742247_1423 (size=12301) 2024-11-28T03:42:36,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765356869, max timestamp=1732765356886Num uniques 859; 2024-11-28T03:42:36,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765356869, max timestamp=1732765356886Num uniques 859; 2024-11-28T03:42:36,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765356869, max timestamp=1732765356886Num uniques 859; 2024-11-28T03:42:36,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:36,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:36,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:36,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:36,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:36,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765416959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765416959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765416960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765416960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:37,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:37,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765417087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765417087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765417088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765417089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,151 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#A#compaction#361 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:37,152 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/a7958cc12c344c08ba083e240439c040 is 175, key is test_row_0/A:col10/1732765356084/Put/seqid=0 2024-11-28T03:42:37,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742248_1424 (size=31597) 2024-11-28T03:42:37,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:37,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:37,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/f3452f58f0534aff9db8291718e4e563 2024-11-28T03:42:37,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/9de559bd05434cf1989f0b5abb96a9c3 is 50, key is test_row_0/C:col10/1732765356751/Put/seqid=0 2024-11-28T03:42:37,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742249_1425 (size=12301) 2024-11-28T03:42:37,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/9de559bd05434cf1989f0b5abb96a9c3 2024-11-28T03:42:37,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/874edc99185b4468bd2ff1bf1e1d107f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/874edc99185b4468bd2ff1bf1e1d107f 2024-11-28T03:42:37,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/874edc99185b4468bd2ff1bf1e1d107f, entries=200, sequenceid=302, filesize=39.0 K 2024-11-28T03:42:37,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/f3452f58f0534aff9db8291718e4e563 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/f3452f58f0534aff9db8291718e4e563 2024-11-28T03:42:37,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/f3452f58f0534aff9db8291718e4e563, entries=150, sequenceid=302, filesize=12.0 K 2024-11-28T03:42:37,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/9de559bd05434cf1989f0b5abb96a9c3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9de559bd05434cf1989f0b5abb96a9c3 2024-11-28T03:42:37,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9de559bd05434cf1989f0b5abb96a9c3, entries=150, sequenceid=302, filesize=12.0 K 2024-11-28T03:42:37,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for eb2f0a77050be706d70e7ee7532fcde6 in 513ms, sequenceid=302, compaction requested=false 2024-11-28T03:42:37,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:37,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-28T03:42:37,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765356921, max timestamp=1732765357310Num uniques 1018; 2024-11-28T03:42:37,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T03:42:37,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732765356921, max timestamp=1732765357310Num uniques 1018; 2024-11-28T03:42:37,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T03:42:37,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:37,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:37,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:37,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:37,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:37,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T03:42:37,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112863575a7893d3413c84e0831a16ade87e_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765356958/Put/seqid=0 2024-11-28T03:42:37,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742250_1426 (size=17534) 2024-11-28T03:42:37,332 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:37,336 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112863575a7893d3413c84e0831a16ade87e_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112863575a7893d3413c84e0831a16ade87e_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:37,337 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/046afd8fe5b549d8aca75818882cf639, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:37,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/046afd8fe5b549d8aca75818882cf639 is 175, key is test_row_0/A:col10/1732765356958/Put/seqid=0 2024-11-28T03:42:37,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742251_1427 (size=48639) 2024-11-28T03:42:37,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765417368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765417381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765417383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765417384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765417487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765417496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765417506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765417506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,565 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/a7958cc12c344c08ba083e240439c040 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/a7958cc12c344c08ba083e240439c040 2024-11-28T03:42:37,569 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/A of eb2f0a77050be706d70e7ee7532fcde6 into a7958cc12c344c08ba083e240439c040(size=30.9 K), total size for store is 69.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:37,569 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:37,570 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/A, priority=13, startTime=1732765356716; duration=0sec 2024-11-28T03:42:37,570 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:37,570 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:A 2024-11-28T03:42:37,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765417696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765417716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765417727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765417727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:37,744 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/046afd8fe5b549d8aca75818882cf639 2024-11-28T03:42:37,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/0ca59e147f6946dd9ee4efd26abeaaca is 50, key is test_row_0/B:col10/1732765356958/Put/seqid=0 2024-11-28T03:42:37,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742252_1428 (size=12301) 2024-11-28T03:42:37,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/0ca59e147f6946dd9ee4efd26abeaaca 2024-11-28T03:42:37,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/65fd94dd0d4d42abb82bbbea004c7bf9 is 50, key is test_row_0/C:col10/1732765356958/Put/seqid=0 2024-11-28T03:42:37,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742253_1429 (size=12301) 2024-11-28T03:42:37,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:37,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:37,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:37,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:37,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:38,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765418016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765418036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765418046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765418047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:38,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:38,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:38,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:38,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:38,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:38,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/65fd94dd0d4d42abb82bbbea004c7bf9 2024-11-28T03:42:38,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/046afd8fe5b549d8aca75818882cf639 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/046afd8fe5b549d8aca75818882cf639 2024-11-28T03:42:38,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/046afd8fe5b549d8aca75818882cf639, entries=250, sequenceid=343, filesize=47.5 K 2024-11-28T03:42:38,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/0ca59e147f6946dd9ee4efd26abeaaca as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/0ca59e147f6946dd9ee4efd26abeaaca 2024-11-28T03:42:38,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/0ca59e147f6946dd9ee4efd26abeaaca, entries=150, sequenceid=343, filesize=12.0 K 2024-11-28T03:42:38,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/65fd94dd0d4d42abb82bbbea004c7bf9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/65fd94dd0d4d42abb82bbbea004c7bf9 2024-11-28T03:42:38,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/65fd94dd0d4d42abb82bbbea004c7bf9, entries=150, sequenceid=343, filesize=12.0 K 2024-11-28T03:42:38,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=107.34 KB/109920 for eb2f0a77050be706d70e7ee7532fcde6 in 873ms, sequenceid=343, compaction requested=true 2024-11-28T03:42:38,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:38,187 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:38,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:38,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:38,188 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:38,188 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:38,188 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/A is initiating minor compaction (all files) 2024-11-28T03:42:38,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:38,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:38,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:38,188 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/A in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:38,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:38,189 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/a7958cc12c344c08ba083e240439c040, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/874edc99185b4468bd2ff1bf1e1d107f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/046afd8fe5b549d8aca75818882cf639] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=117.4 K 2024-11-28T03:42:38,189 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:38,189 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/a7958cc12c344c08ba083e240439c040, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/874edc99185b4468bd2ff1bf1e1d107f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/046afd8fe5b549d8aca75818882cf639] 2024-11-28T03:42:38,190 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7958cc12c344c08ba083e240439c040, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765356084 2024-11-28T03:42:38,190 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:38,190 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/B is initiating minor compaction (all files) 2024-11-28T03:42:38,190 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/B in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:38,190 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b5823df254ca460ca4a9412b9d7ed74a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/f3452f58f0534aff9db8291718e4e563, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/0ca59e147f6946dd9ee4efd26abeaaca] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=36.4 K 2024-11-28T03:42:38,190 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 874edc99185b4468bd2ff1bf1e1d107f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732765356216 2024-11-28T03:42:38,191 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting b5823df254ca460ca4a9412b9d7ed74a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765356084 2024-11-28T03:42:38,191 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 046afd8fe5b549d8aca75818882cf639, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732765356921 2024-11-28T03:42:38,191 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting f3452f58f0534aff9db8291718e4e563, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732765356216 2024-11-28T03:42:38,192 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca59e147f6946dd9ee4efd26abeaaca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732765356958 2024-11-28T03:42:38,196 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:38,206 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:38,207 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/8d0bede7f0114fdda18966a112f164bc is 50, key is test_row_0/B:col10/1732765356958/Put/seqid=0 2024-11-28T03:42:38,208 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411280369641029dd43b49e221e44afff6518_eb2f0a77050be706d70e7ee7532fcde6 store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:38,210 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411280369641029dd43b49e221e44afff6518_eb2f0a77050be706d70e7ee7532fcde6, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:38,210 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280369641029dd43b49e221e44afff6518_eb2f0a77050be706d70e7ee7532fcde6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:38,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742254_1430 (size=12745) 2024-11-28T03:42:38,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742255_1431 (size=4469) 2024-11-28T03:42:38,219 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#A#compaction#369 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:38,219 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/f16d04538a934f90839c429b6b3491dd is 175, key is test_row_0/A:col10/1732765356958/Put/seqid=0 2024-11-28T03:42:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742256_1432 (size=31699) 2024-11-28T03:42:38,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T03:42:38,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:38,302 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:42:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:38,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128863c055f0d7f48c4bd671eea8dc9026f_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765357354/Put/seqid=0 2024-11-28T03:42:38,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742257_1433 (size=12454) 2024-11-28T03:42:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:38,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:38,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765418580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765418585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765418589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765418590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,621 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/8d0bede7f0114fdda18966a112f164bc as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8d0bede7f0114fdda18966a112f164bc 2024-11-28T03:42:38,625 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/B of eb2f0a77050be706d70e7ee7532fcde6 into 8d0bede7f0114fdda18966a112f164bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:38,626 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:38,626 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/B, priority=13, startTime=1732765358188; duration=0sec 2024-11-28T03:42:38,626 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:38,626 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:B 2024-11-28T03:42:38,626 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T03:42:38,627 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:38,627 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/C is initiating minor compaction (all files) 2024-11-28T03:42:38,627 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/C in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:38,627 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/b3dc511c2cb54750aa374ebcf01b9b24, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9de559bd05434cf1989f0b5abb96a9c3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/65fd94dd0d4d42abb82bbbea004c7bf9] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=36.4 K 2024-11-28T03:42:38,628 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting b3dc511c2cb54750aa374ebcf01b9b24, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732765356084 2024-11-28T03:42:38,628 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de559bd05434cf1989f0b5abb96a9c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732765356216 2024-11-28T03:42:38,629 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 65fd94dd0d4d42abb82bbbea004c7bf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732765356958 2024-11-28T03:42:38,630 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/f16d04538a934f90839c429b6b3491dd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/f16d04538a934f90839c429b6b3491dd 2024-11-28T03:42:38,637 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/A of eb2f0a77050be706d70e7ee7532fcde6 into f16d04538a934f90839c429b6b3491dd(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:38,637 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:38,637 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/A, priority=13, startTime=1732765358187; duration=0sec 2024-11-28T03:42:38,637 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:38,637 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:A 2024-11-28T03:42:38,641 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#C#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:38,642 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/c5a62438a2fb4d11a98c6411f3817219 is 50, key is test_row_0/C:col10/1732765356958/Put/seqid=0 2024-11-28T03:42:38,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742258_1434 (size=12745) 2024-11-28T03:42:38,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765418697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765418697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,700 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in eb2f0a77050be706d70e7ee7532fcde6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765418706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765418706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765418707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:38,725 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128863c055f0d7f48c4bd671eea8dc9026f_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128863c055f0d7f48c4bd671eea8dc9026f_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/357947f0b7ba40bfbd7ea86947f406c5, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:38,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/357947f0b7ba40bfbd7ea86947f406c5 is 175, key is test_row_0/A:col10/1732765357354/Put/seqid=0 2024-11-28T03:42:38,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742259_1435 (size=31255) 2024-11-28T03:42:38,733 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/357947f0b7ba40bfbd7ea86947f406c5 2024-11-28T03:42:38,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/55605e52ad124fbaba01140c03a683c4 is 50, key is test_row_0/B:col10/1732765357354/Put/seqid=0 2024-11-28T03:42:38,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742260_1436 (size=12301) 2024-11-28T03:42:38,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765418906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765418927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765418927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:38,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765418927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,052 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/c5a62438a2fb4d11a98c6411f3817219 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/c5a62438a2fb4d11a98c6411f3817219 2024-11-28T03:42:39,057 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/C of eb2f0a77050be706d70e7ee7532fcde6 into c5a62438a2fb4d11a98c6411f3817219(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:39,057 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:39,057 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/C, priority=13, startTime=1732765358188; duration=0sec 2024-11-28T03:42:39,058 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:39,058 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:C 2024-11-28T03:42:39,149 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/55605e52ad124fbaba01140c03a683c4 2024-11-28T03:42:39,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/7fe633c04c2846f1bb1cd9b4c9d0a78f is 50, key is test_row_0/C:col10/1732765357354/Put/seqid=0 2024-11-28T03:42:39,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742261_1437 (size=12301) 2024-11-28T03:42:39,161 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/7fe633c04c2846f1bb1cd9b4c9d0a78f 2024-11-28T03:42:39,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/357947f0b7ba40bfbd7ea86947f406c5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/357947f0b7ba40bfbd7ea86947f406c5 2024-11-28T03:42:39,179 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/357947f0b7ba40bfbd7ea86947f406c5, entries=150, sequenceid=363, filesize=30.5 K 2024-11-28T03:42:39,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/55605e52ad124fbaba01140c03a683c4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/55605e52ad124fbaba01140c03a683c4 2024-11-28T03:42:39,183 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/55605e52ad124fbaba01140c03a683c4, entries=150, sequenceid=363, filesize=12.0 K 2024-11-28T03:42:39,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/7fe633c04c2846f1bb1cd9b4c9d0a78f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7fe633c04c2846f1bb1cd9b4c9d0a78f 2024-11-28T03:42:39,187 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7fe633c04c2846f1bb1cd9b4c9d0a78f, entries=150, sequenceid=363, filesize=12.0 K 2024-11-28T03:42:39,188 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for eb2f0a77050be706d70e7ee7532fcde6 in 887ms, sequenceid=363, compaction requested=false 2024-11-28T03:42:39,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:39,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:39,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-28T03:42:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-28T03:42:39,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-28T03:42:39,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.7300 sec 2024-11-28T03:42:39,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 4.7350 sec 2024-11-28T03:42:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:39,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:42:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:39,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d7fe867076d44186839c4e81d9e721b3_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765358583/Put/seqid=0 2024-11-28T03:42:39,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742262_1438 (size=14994) 2024-11-28T03:42:39,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765419272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765419273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765419274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765419278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765419387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765419396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765419396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765419396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765419596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765419606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765419606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765419606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,644 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:39,648 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d7fe867076d44186839c4e81d9e721b3_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d7fe867076d44186839c4e81d9e721b3_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:39,649 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/497314f0c795400c969a8be2bba64d3c, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:39,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/497314f0c795400c969a8be2bba64d3c is 175, key is test_row_0/A:col10/1732765358583/Put/seqid=0 2024-11-28T03:42:39,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742263_1439 (size=39949) 2024-11-28T03:42:39,654 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/497314f0c795400c969a8be2bba64d3c 2024-11-28T03:42:39,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/1bfc6610e64a4052bf8730c967f594bd is 50, key is test_row_0/B:col10/1732765358583/Put/seqid=0 2024-11-28T03:42:39,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742264_1440 (size=12301) 2024-11-28T03:42:39,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765419916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765419926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765419926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765419926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/1bfc6610e64a4052bf8730c967f594bd 2024-11-28T03:42:40,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/1b04cc7397174977b2a87a4d7f85d006 is 50, key is test_row_0/C:col10/1732765358583/Put/seqid=0 2024-11-28T03:42:40,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742265_1441 (size=12301) 2024-11-28T03:42:40,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/1b04cc7397174977b2a87a4d7f85d006 2024-11-28T03:42:40,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/497314f0c795400c969a8be2bba64d3c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/497314f0c795400c969a8be2bba64d3c 2024-11-28T03:42:40,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/497314f0c795400c969a8be2bba64d3c, entries=200, sequenceid=380, filesize=39.0 K 2024-11-28T03:42:40,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/1bfc6610e64a4052bf8730c967f594bd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1bfc6610e64a4052bf8730c967f594bd 2024-11-28T03:42:40,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1bfc6610e64a4052bf8730c967f594bd, entries=150, sequenceid=380, filesize=12.0 K 2024-11-28T03:42:40,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/1b04cc7397174977b2a87a4d7f85d006 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1b04cc7397174977b2a87a4d7f85d006 2024-11-28T03:42:40,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1b04cc7397174977b2a87a4d7f85d006, entries=150, sequenceid=380, filesize=12.0 K 2024-11-28T03:42:40,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for eb2f0a77050be706d70e7ee7532fcde6 in 872ms, sequenceid=380, compaction requested=true 2024-11-28T03:42:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:40,103 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:40,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:40,103 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:40,104 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:40,104 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:40,104 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/A is initiating minor compaction (all files) 2024-11-28T03:42:40,104 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/B is initiating minor compaction (all files) 2024-11-28T03:42:40,104 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/A in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:40,105 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/B in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:40,105 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/f16d04538a934f90839c429b6b3491dd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/357947f0b7ba40bfbd7ea86947f406c5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/497314f0c795400c969a8be2bba64d3c] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=100.5 K 2024-11-28T03:42:40,105 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8d0bede7f0114fdda18966a112f164bc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/55605e52ad124fbaba01140c03a683c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1bfc6610e64a4052bf8730c967f594bd] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=36.5 K 2024-11-28T03:42:40,105 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:40,105 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/f16d04538a934f90839c429b6b3491dd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/357947f0b7ba40bfbd7ea86947f406c5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/497314f0c795400c969a8be2bba64d3c] 2024-11-28T03:42:40,105 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f16d04538a934f90839c429b6b3491dd, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732765356958 2024-11-28T03:42:40,105 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d0bede7f0114fdda18966a112f164bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732765356958 2024-11-28T03:42:40,105 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 55605e52ad124fbaba01140c03a683c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732765357354 2024-11-28T03:42:40,105 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 357947f0b7ba40bfbd7ea86947f406c5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732765357354 2024-11-28T03:42:40,106 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 497314f0c795400c969a8be2bba64d3c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732765358583 2024-11-28T03:42:40,106 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bfc6610e64a4052bf8730c967f594bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732765358583 2024-11-28T03:42:40,120 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:40,120 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/4d0660f66e8f4801a0c5b390d5487680 is 50, key is test_row_0/B:col10/1732765358583/Put/seqid=0 2024-11-28T03:42:40,121 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:40,129 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411282a45d93f909543e78b673d0aae334546_eb2f0a77050be706d70e7ee7532fcde6 store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:40,131 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411282a45d93f909543e78b673d0aae334546_eb2f0a77050be706d70e7ee7532fcde6, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:40,131 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282a45d93f909543e78b673d0aae334546_eb2f0a77050be706d70e7ee7532fcde6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:40,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742266_1442 (size=12847) 2024-11-28T03:42:40,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742267_1443 (size=4469) 2024-11-28T03:42:40,147 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#A#compaction#379 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:40,148 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/da18c15cc9c44bf08657df218b153204 is 175, key is test_row_0/A:col10/1732765358583/Put/seqid=0 2024-11-28T03:42:40,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742268_1444 (size=31801) 2024-11-28T03:42:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:40,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:42:40,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:40,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:40,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:40,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d10a9e51ff244a5982db188a219c6b2c_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765360438/Put/seqid=0 2024-11-28T03:42:40,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742269_1445 (size=14994) 2024-11-28T03:42:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765420468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765420468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765420469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765420470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,539 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/4d0660f66e8f4801a0c5b390d5487680 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4d0660f66e8f4801a0c5b390d5487680 2024-11-28T03:42:40,544 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/B of eb2f0a77050be706d70e7ee7532fcde6 into 4d0660f66e8f4801a0c5b390d5487680(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:40,544 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:40,544 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/B, priority=13, startTime=1732765360103; duration=0sec 2024-11-28T03:42:40,544 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:40,544 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:B 2024-11-28T03:42:40,544 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:40,545 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:40,545 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/C is initiating minor compaction (all files) 2024-11-28T03:42:40,545 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/C in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:40,545 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/c5a62438a2fb4d11a98c6411f3817219, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7fe633c04c2846f1bb1cd9b4c9d0a78f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1b04cc7397174977b2a87a4d7f85d006] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=36.5 K 2024-11-28T03:42:40,545 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting c5a62438a2fb4d11a98c6411f3817219, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732765356958 2024-11-28T03:42:40,546 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe633c04c2846f1bb1cd9b4c9d0a78f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732765357354 2024-11-28T03:42:40,546 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b04cc7397174977b2a87a4d7f85d006, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732765358583 2024-11-28T03:42:40,552 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:40,552 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/a301d35aa1264fcda47c33c5db95f24c is 50, key is test_row_0/C:col10/1732765358583/Put/seqid=0 2024-11-28T03:42:40,560 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/da18c15cc9c44bf08657df218b153204 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/da18c15cc9c44bf08657df218b153204 2024-11-28T03:42:40,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742270_1446 (size=12847) 2024-11-28T03:42:40,566 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/A of eb2f0a77050be706d70e7ee7532fcde6 into da18c15cc9c44bf08657df218b153204(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:40,566 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:40,566 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/A, priority=13, startTime=1732765360103; duration=0sec 2024-11-28T03:42:40,566 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:40,566 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:A 2024-11-28T03:42:40,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765420586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765420586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765420587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765420597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765420806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765420807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765420807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765420807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:40,855 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:40,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d10a9e51ff244a5982db188a219c6b2c_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d10a9e51ff244a5982db188a219c6b2c_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:40,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/b1592af444fd4dfab2ba3d3737555099, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:40,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/b1592af444fd4dfab2ba3d3737555099 is 175, key is test_row_0/A:col10/1732765360438/Put/seqid=0 2024-11-28T03:42:40,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742271_1447 (size=39949) 2024-11-28T03:42:40,867 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/b1592af444fd4dfab2ba3d3737555099 2024-11-28T03:42:40,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/11b80f7316db450db3b8a3e50399d469 is 50, key is test_row_0/B:col10/1732765360438/Put/seqid=0 2024-11-28T03:42:40,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742272_1448 (size=12301) 2024-11-28T03:42:40,971 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/a301d35aa1264fcda47c33c5db95f24c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/a301d35aa1264fcda47c33c5db95f24c 2024-11-28T03:42:40,975 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/C of eb2f0a77050be706d70e7ee7532fcde6 into a301d35aa1264fcda47c33c5db95f24c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:40,975 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:40,975 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/C, priority=13, startTime=1732765360103; duration=0sec 2024-11-28T03:42:40,975 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:40,975 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:C 2024-11-28T03:42:41,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765421116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765421116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765421127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765421127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/11b80f7316db450db3b8a3e50399d469 2024-11-28T03:42:41,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/016a4a26ad3444b6a7fd4f7337634ac3 is 50, key is test_row_0/C:col10/1732765360438/Put/seqid=0 2024-11-28T03:42:41,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742273_1449 (size=12301) 2024-11-28T03:42:41,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/016a4a26ad3444b6a7fd4f7337634ac3 2024-11-28T03:42:41,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/b1592af444fd4dfab2ba3d3737555099 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/b1592af444fd4dfab2ba3d3737555099 2024-11-28T03:42:41,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/b1592af444fd4dfab2ba3d3737555099, entries=200, sequenceid=399, filesize=39.0 K 2024-11-28T03:42:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/11b80f7316db450db3b8a3e50399d469 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/11b80f7316db450db3b8a3e50399d469 2024-11-28T03:42:41,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/11b80f7316db450db3b8a3e50399d469, entries=150, sequenceid=399, filesize=12.0 K 2024-11-28T03:42:41,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/016a4a26ad3444b6a7fd4f7337634ac3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/016a4a26ad3444b6a7fd4f7337634ac3 2024-11-28T03:42:41,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/016a4a26ad3444b6a7fd4f7337634ac3, entries=150, sequenceid=399, filesize=12.0 K 2024-11-28T03:42:41,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for eb2f0a77050be706d70e7ee7532fcde6 in 890ms, sequenceid=399, compaction requested=false 2024-11-28T03:42:41,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:41,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:42:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:41,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c4d02a79f5044b7d9337b4769d23c30b_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765360466/Put/seqid=0 2024-11-28T03:42:41,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742274_1450 (size=14994) 2024-11-28T03:42:41,656 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:41,659 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c4d02a79f5044b7d9337b4769d23c30b_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c4d02a79f5044b7d9337b4769d23c30b_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:41,661 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/7ab8a82dfc644f57b740d3ae619df2bf, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:41,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/7ab8a82dfc644f57b740d3ae619df2bf is 175, key is test_row_0/A:col10/1732765360466/Put/seqid=0 2024-11-28T03:42:41,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742275_1451 (size=39949) 2024-11-28T03:42:41,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/7ab8a82dfc644f57b740d3ae619df2bf 2024-11-28T03:42:41,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/8f5ca5d0489f425da23db0b93305caae is 50, key is test_row_0/B:col10/1732765360466/Put/seqid=0 2024-11-28T03:42:41,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742276_1452 (size=12301) 2024-11-28T03:42:41,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765421682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765421686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765421687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765421691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765421806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765421806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765421807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:41,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765421816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765422026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765422027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765422027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765422027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/8f5ca5d0489f425da23db0b93305caae 2024-11-28T03:42:42,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/83343608da464e1aa79a02a201f847ae is 50, key is test_row_0/C:col10/1732765360466/Put/seqid=0 2024-11-28T03:42:42,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742277_1453 (size=12301) 2024-11-28T03:42:42,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765422336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765422336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765422336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765422349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/83343608da464e1aa79a02a201f847ae 2024-11-28T03:42:42,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/7ab8a82dfc644f57b740d3ae619df2bf as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7ab8a82dfc644f57b740d3ae619df2bf 2024-11-28T03:42:42,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7ab8a82dfc644f57b740d3ae619df2bf, entries=200, sequenceid=416, filesize=39.0 K 2024-11-28T03:42:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/8f5ca5d0489f425da23db0b93305caae as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8f5ca5d0489f425da23db0b93305caae 2024-11-28T03:42:42,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8f5ca5d0489f425da23db0b93305caae, entries=150, sequenceid=416, filesize=12.0 K 2024-11-28T03:42:42,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/83343608da464e1aa79a02a201f847ae as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/83343608da464e1aa79a02a201f847ae 2024-11-28T03:42:42,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/83343608da464e1aa79a02a201f847ae, entries=150, sequenceid=416, filesize=12.0 K 2024-11-28T03:42:42,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for eb2f0a77050be706d70e7ee7532fcde6 in 868ms, sequenceid=416, compaction requested=true 2024-11-28T03:42:42,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:42,510 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:42,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:42,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:42,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:42,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:42,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:42,510 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:42,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:42,511 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:42,511 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/A is initiating minor compaction (all files) 2024-11-28T03:42:42,511 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/A in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:42,511 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/da18c15cc9c44bf08657df218b153204, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/b1592af444fd4dfab2ba3d3737555099, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7ab8a82dfc644f57b740d3ae619df2bf] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=109.1 K 2024-11-28T03:42:42,511 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:42,511 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/da18c15cc9c44bf08657df218b153204, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/b1592af444fd4dfab2ba3d3737555099, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7ab8a82dfc644f57b740d3ae619df2bf] 2024-11-28T03:42:42,511 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:42,511 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting da18c15cc9c44bf08657df218b153204, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732765358583 2024-11-28T03:42:42,511 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/B is initiating minor compaction (all files) 2024-11-28T03:42:42,512 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/B in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:42,512 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4d0660f66e8f4801a0c5b390d5487680, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/11b80f7316db450db3b8a3e50399d469, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8f5ca5d0489f425da23db0b93305caae] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=36.6 K 2024-11-28T03:42:42,512 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1592af444fd4dfab2ba3d3737555099, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732765359270 2024-11-28T03:42:42,512 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ab8a82dfc644f57b740d3ae619df2bf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732765360455 2024-11-28T03:42:42,512 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d0660f66e8f4801a0c5b390d5487680, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732765358583 2024-11-28T03:42:42,512 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b80f7316db450db3b8a3e50399d469, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732765359270 2024-11-28T03:42:42,513 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5ca5d0489f425da23db0b93305caae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732765360455 2024-11-28T03:42:42,524 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:42,526 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:42,526 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/63dbf37bd4ec4c15a3b9dc07707dee8e is 50, key is test_row_0/B:col10/1732765360466/Put/seqid=0 2024-11-28T03:42:42,531 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128234d9177d39c4c3bacc5b02e3911a7f2_eb2f0a77050be706d70e7ee7532fcde6 store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:42,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742278_1454 (size=12949) 2024-11-28T03:42:42,532 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128234d9177d39c4c3bacc5b02e3911a7f2_eb2f0a77050be706d70e7ee7532fcde6, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:42,532 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128234d9177d39c4c3bacc5b02e3911a7f2_eb2f0a77050be706d70e7ee7532fcde6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:42,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742279_1455 (size=4469) 2024-11-28T03:42:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T03:42:42,667 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:42,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-28T03:42:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T03:42:42,670 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:42,671 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:42,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:42,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:42:42,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:42,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:42,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:42,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:42,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:42,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:42,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e447be7aa85d4c0d827238c477c58b38_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765361690/Put/seqid=0 2024-11-28T03:42:42,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742280_1456 (size=14994) 2024-11-28T03:42:42,747 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:42,751 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e447be7aa85d4c0d827238c477c58b38_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e447be7aa85d4c0d827238c477c58b38_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:42,752 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/e8d296accc3b4c128436d8f4eb82d8a8, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:42,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/e8d296accc3b4c128436d8f4eb82d8a8 is 175, key is test_row_0/A:col10/1732765361690/Put/seqid=0 2024-11-28T03:42:42,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742281_1457 (size=39949) 2024-11-28T03:42:42,760 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/e8d296accc3b4c128436d8f4eb82d8a8 2024-11-28T03:42:42,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/6d355f32c9034291bfa9669921c788c2 is 50, key is test_row_0/B:col10/1732765361690/Put/seqid=0 2024-11-28T03:42:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T03:42:42,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742282_1458 (size=12301) 2024-11-28T03:42:42,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T03:42:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:42,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:42,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765422832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765422846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765422846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765422856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765422867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,937 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/63dbf37bd4ec4c15a3b9dc07707dee8e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/63dbf37bd4ec4c15a3b9dc07707dee8e 2024-11-28T03:42:42,937 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#A#compaction#387 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:42,938 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/9108614a124143c9a7e962206259e387 is 175, key is test_row_0/A:col10/1732765360466/Put/seqid=0 2024-11-28T03:42:42,943 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/B of eb2f0a77050be706d70e7ee7532fcde6 into 63dbf37bd4ec4c15a3b9dc07707dee8e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:42,943 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:42,943 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/B, priority=13, startTime=1732765362510; duration=0sec 2024-11-28T03:42:42,943 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:42,943 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:B 2024-11-28T03:42:42,943 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:42,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742283_1459 (size=31903) 2024-11-28T03:42:42,945 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:42,945 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/C is initiating minor compaction (all files) 2024-11-28T03:42:42,945 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/C in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:42,945 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/a301d35aa1264fcda47c33c5db95f24c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/016a4a26ad3444b6a7fd4f7337634ac3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/83343608da464e1aa79a02a201f847ae] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=36.6 K 2024-11-28T03:42:42,946 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting a301d35aa1264fcda47c33c5db95f24c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732765358583 2024-11-28T03:42:42,946 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 016a4a26ad3444b6a7fd4f7337634ac3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732765359270 2024-11-28T03:42:42,947 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 83343608da464e1aa79a02a201f847ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732765360455 2024-11-28T03:42:42,951 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/9108614a124143c9a7e962206259e387 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/9108614a124143c9a7e962206259e387 2024-11-28T03:42:42,955 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/A of eb2f0a77050be706d70e7ee7532fcde6 into 9108614a124143c9a7e962206259e387(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:42,955 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:42,955 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/A, priority=13, startTime=1732765362510; duration=0sec 2024-11-28T03:42:42,955 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:42,955 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:A 2024-11-28T03:42:42,957 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#C#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:42,958 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/36dfae2124a04b45a88b35825988128c is 50, key is test_row_0/C:col10/1732765360466/Put/seqid=0 2024-11-28T03:42:42,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742284_1460 (size=12949) 2024-11-28T03:42:42,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765422946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:42,967 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/36dfae2124a04b45a88b35825988128c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/36dfae2124a04b45a88b35825988128c 2024-11-28T03:42:42,970 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/C of eb2f0a77050be706d70e7ee7532fcde6 into 36dfae2124a04b45a88b35825988128c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:42,970 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:42,970 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/C, priority=13, startTime=1732765362510; duration=0sec 2024-11-28T03:42:42,970 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:42,970 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:C 2024-11-28T03:42:42,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T03:42:42,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:42,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:42,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:42,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:42,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T03:42:43,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T03:42:43,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:43,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:43,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:43,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:43,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:43,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765423166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:43,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/6d355f32c9034291bfa9669921c788c2 2024-11-28T03:42:43,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/9f2b06b19a7c43948cc34587e04c6099 is 50, key is test_row_0/C:col10/1732765361690/Put/seqid=0 2024-11-28T03:42:43,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742285_1461 (size=12301) 2024-11-28T03:42:43,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/9f2b06b19a7c43948cc34587e04c6099 2024-11-28T03:42:43,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/e8d296accc3b4c128436d8f4eb82d8a8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/e8d296accc3b4c128436d8f4eb82d8a8 2024-11-28T03:42:43,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/e8d296accc3b4c128436d8f4eb82d8a8, entries=200, sequenceid=435, filesize=39.0 K 2024-11-28T03:42:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/6d355f32c9034291bfa9669921c788c2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/6d355f32c9034291bfa9669921c788c2 2024-11-28T03:42:43,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/6d355f32c9034291bfa9669921c788c2, entries=150, sequenceid=435, filesize=12.0 K 2024-11-28T03:42:43,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/9f2b06b19a7c43948cc34587e04c6099 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9f2b06b19a7c43948cc34587e04c6099 2024-11-28T03:42:43,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9f2b06b19a7c43948cc34587e04c6099, entries=150, sequenceid=435, filesize=12.0 K 2024-11-28T03:42:43,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for eb2f0a77050be706d70e7ee7532fcde6 in 509ms, sequenceid=435, compaction requested=false 2024-11-28T03:42:43,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:43,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T03:42:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:43,286 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T03:42:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:43,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128636cea58c53a450580a99c425b334f51_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765362830/Put/seqid=0 2024-11-28T03:42:43,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742286_1462 (size=12454) 2024-11-28T03:42:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T03:42:43,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:43,300 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128636cea58c53a450580a99c425b334f51_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128636cea58c53a450580a99c425b334f51_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:43,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/db00480f4f9c41e3bd9a6140cb42b1ea, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:43,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/db00480f4f9c41e3bd9a6140cb42b1ea is 175, key is test_row_0/A:col10/1732765362830/Put/seqid=0 2024-11-28T03:42:43,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742287_1463 (size=31255) 2024-11-28T03:42:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:43,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765423610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:43,706 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=451, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/db00480f4f9c41e3bd9a6140cb42b1ea 2024-11-28T03:42:43,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/b34fe0ea3492411d83a0e19daa2d7e08 is 50, key is test_row_0/B:col10/1732765362830/Put/seqid=0 2024-11-28T03:42:43,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742288_1464 (size=12301) 2024-11-28T03:42:43,718 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/b34fe0ea3492411d83a0e19daa2d7e08 2024-11-28T03:42:43,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/11559eb8ed3e4b25853199a17895990a is 50, key is test_row_0/C:col10/1732765362830/Put/seqid=0 2024-11-28T03:42:43,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742289_1465 (size=12301) 2024-11-28T03:42:43,728 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/11559eb8ed3e4b25853199a17895990a 2024-11-28T03:42:43,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/db00480f4f9c41e3bd9a6140cb42b1ea as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/db00480f4f9c41e3bd9a6140cb42b1ea 2024-11-28T03:42:43,735 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/db00480f4f9c41e3bd9a6140cb42b1ea, entries=150, sequenceid=451, filesize=30.5 K 2024-11-28T03:42:43,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/b34fe0ea3492411d83a0e19daa2d7e08 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b34fe0ea3492411d83a0e19daa2d7e08 2024-11-28T03:42:43,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765423727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:43,740 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b34fe0ea3492411d83a0e19daa2d7e08, entries=150, sequenceid=451, filesize=12.0 K 2024-11-28T03:42:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/11559eb8ed3e4b25853199a17895990a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/11559eb8ed3e4b25853199a17895990a 2024-11-28T03:42:43,743 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/11559eb8ed3e4b25853199a17895990a, entries=150, sequenceid=451, filesize=12.0 K 2024-11-28T03:42:43,744 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for eb2f0a77050be706d70e7ee7532fcde6 in 458ms, sequenceid=451, compaction requested=true 2024-11-28T03:42:43,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:43,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:43,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-28T03:42:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-28T03:42:43,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-28T03:42:43,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0750 sec 2024-11-28T03:42:43,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0810 sec 2024-11-28T03:42:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T03:42:43,809 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:43,810 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-28T03:42:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T03:42:43,813 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:43,814 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:43,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:43,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T03:42:43,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:43,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:43,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:43,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:43,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:43,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T03:42:43,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288acfa0ffc95e4e6cbe63722a8973e6d3_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765363599/Put/seqid=0 2024-11-28T03:42:43,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765423913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765423933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:43,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742290_1466 (size=12454) 2024-11-28T03:42:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:43,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765423936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765423937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:43,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765423947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:43,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:43,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:43,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:43,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:43,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:43,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765424046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765424057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765424057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765424057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:44,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:44,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T03:42:44,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765424256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:44,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:44,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765424276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765424276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765424287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765424287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,346 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:44,350 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288acfa0ffc95e4e6cbe63722a8973e6d3_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288acfa0ffc95e4e6cbe63722a8973e6d3_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:44,351 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/7cdd721b78bc45229d382b224c99c577, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:44,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/7cdd721b78bc45229d382b224c99c577 is 175, key is test_row_0/A:col10/1732765363599/Put/seqid=0 2024-11-28T03:42:44,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742291_1467 (size=31255) 2024-11-28T03:42:44,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:44,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:44,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T03:42:44,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:44,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:44,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765424576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765424597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765424606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765424606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:44,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,760 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=472, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/7cdd721b78bc45229d382b224c99c577 2024-11-28T03:42:44,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/fcb44397dc88461d84f40d30ed9c127e is 50, key is test_row_0/B:col10/1732765363599/Put/seqid=0 2024-11-28T03:42:44,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742292_1468 (size=12301) 2024-11-28T03:42:44,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765424797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:44,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:44,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:44,895 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38327 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3a546af21488,34671,1732765249447, table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T03:42:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T03:42:45,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:45,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765425096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765425106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:45,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765425127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:45,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765425127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:45,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/fcb44397dc88461d84f40d30ed9c127e 2024-11-28T03:42:45,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/d56ce54268714712915cf0232566488c is 50, key is test_row_0/C:col10/1732765363599/Put/seqid=0 2024-11-28T03:42:45,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742293_1469 (size=12301) 2024-11-28T03:42:45,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:45,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:45,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:45,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/d56ce54268714712915cf0232566488c 2024-11-28T03:42:45,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/7cdd721b78bc45229d382b224c99c577 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7cdd721b78bc45229d382b224c99c577 2024-11-28T03:42:45,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7cdd721b78bc45229d382b224c99c577, entries=150, sequenceid=472, filesize=30.5 K 2024-11-28T03:42:45,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/fcb44397dc88461d84f40d30ed9c127e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/fcb44397dc88461d84f40d30ed9c127e 2024-11-28T03:42:45,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/fcb44397dc88461d84f40d30ed9c127e, entries=150, sequenceid=472, filesize=12.0 K 2024-11-28T03:42:45,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/d56ce54268714712915cf0232566488c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d56ce54268714712915cf0232566488c 2024-11-28T03:42:45,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d56ce54268714712915cf0232566488c, entries=150, sequenceid=472, filesize=12.0 K 2024-11-28T03:42:45,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for eb2f0a77050be706d70e7ee7532fcde6 in 1717ms, sequenceid=472, compaction requested=true 2024-11-28T03:42:45,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:45,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:45,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:45,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:45,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:45,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:45,604 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:42:45,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T03:42:45,604 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:42:45,605 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:42:45,605 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/A is initiating minor compaction (all files) 2024-11-28T03:42:45,605 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:42:45,605 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/A in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,606 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/B is initiating minor compaction (all files) 2024-11-28T03:42:45,606 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/9108614a124143c9a7e962206259e387, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/e8d296accc3b4c128436d8f4eb82d8a8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/db00480f4f9c41e3bd9a6140cb42b1ea, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7cdd721b78bc45229d382b224c99c577] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=131.2 K 2024-11-28T03:42:45,606 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/B in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,606 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,606 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/63dbf37bd4ec4c15a3b9dc07707dee8e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/6d355f32c9034291bfa9669921c788c2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b34fe0ea3492411d83a0e19daa2d7e08, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/fcb44397dc88461d84f40d30ed9c127e] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=48.7 K 2024-11-28T03:42:45,606 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/9108614a124143c9a7e962206259e387, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/e8d296accc3b4c128436d8f4eb82d8a8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/db00480f4f9c41e3bd9a6140cb42b1ea, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7cdd721b78bc45229d382b224c99c577] 2024-11-28T03:42:45,608 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 63dbf37bd4ec4c15a3b9dc07707dee8e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732765360455 2024-11-28T03:42:45,609 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9108614a124143c9a7e962206259e387, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732765360455 2024-11-28T03:42:45,609 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d355f32c9034291bfa9669921c788c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732765361686 2024-11-28T03:42:45,609 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d296accc3b4c128436d8f4eb82d8a8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732765361671 2024-11-28T03:42:45,610 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting db00480f4f9c41e3bd9a6140cb42b1ea, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732765362787 2024-11-28T03:42:45,610 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting b34fe0ea3492411d83a0e19daa2d7e08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732765362787 2024-11-28T03:42:45,610 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cdd721b78bc45229d382b224c99c577, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732765363599 2024-11-28T03:42:45,610 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb44397dc88461d84f40d30ed9c127e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732765363599 2024-11-28T03:42:45,619 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#B#compaction#399 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:45,620 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/dc14de9c217746aba7dc0ed51e0ac7bb is 50, key is test_row_0/B:col10/1732765363599/Put/seqid=0 2024-11-28T03:42:45,627 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:45,630 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128243569c6a6f04759b0e77276ef0f5309_eb2f0a77050be706d70e7ee7532fcde6 store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:45,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742294_1470 (size=13085) 2024-11-28T03:42:45,633 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128243569c6a6f04759b0e77276ef0f5309_eb2f0a77050be706d70e7ee7532fcde6, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:45,633 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128243569c6a6f04759b0e77276ef0f5309_eb2f0a77050be706d70e7ee7532fcde6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:45,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742295_1471 (size=4469) 2024-11-28T03:42:45,644 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#A#compaction#400 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:45,645 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/960c51088a5b421d98c509afb23a6dac is 175, key is test_row_0/A:col10/1732765363599/Put/seqid=0 2024-11-28T03:42:45,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742296_1472 (size=32039) 2024-11-28T03:42:45,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-28T03:42:45,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,657 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T03:42:45,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:45,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:45,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:45,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:45,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:45,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:45,659 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/960c51088a5b421d98c509afb23a6dac as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/960c51088a5b421d98c509afb23a6dac 2024-11-28T03:42:45,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288f987bb13f184328b7bca590b046114e_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765363935/Put/seqid=0 2024-11-28T03:42:45,665 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/A of eb2f0a77050be706d70e7ee7532fcde6 into 960c51088a5b421d98c509afb23a6dac(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:45,665 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:45,665 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/A, priority=12, startTime=1732765365604; duration=0sec 2024-11-28T03:42:45,665 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:45,665 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:A 2024-11-28T03:42:45,665 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:42:45,666 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:42:45,666 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/C is initiating minor compaction (all files) 2024-11-28T03:42:45,666 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/C in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:45,667 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/36dfae2124a04b45a88b35825988128c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9f2b06b19a7c43948cc34587e04c6099, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/11559eb8ed3e4b25853199a17895990a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d56ce54268714712915cf0232566488c] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=48.7 K 2024-11-28T03:42:45,667 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36dfae2124a04b45a88b35825988128c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732765360455 2024-11-28T03:42:45,667 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2b06b19a7c43948cc34587e04c6099, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732765361686 2024-11-28T03:42:45,667 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11559eb8ed3e4b25853199a17895990a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732765362787 2024-11-28T03:42:45,668 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d56ce54268714712915cf0232566488c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732765363599 2024-11-28T03:42:45,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742297_1473 (size=12454) 2024-11-28T03:42:45,679 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:45,680 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/401bc9cd440840518fb66d8fb4f4663e is 50, key is test_row_0/C:col10/1732765363599/Put/seqid=0 2024-11-28T03:42:45,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742298_1474 (size=13085) 2024-11-28T03:42:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:45,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T03:42:45,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765365938, max timestamp=1732765365982Num uniques 859; 2024-11-28T03:42:45,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:45,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765365938, max timestamp=1732765365982Num uniques 859; 2024-11-28T03:42:45,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:45,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765365938, max timestamp=1732765365982Num uniques 859; 2024-11-28T03:42:45,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:42:46,037 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/dc14de9c217746aba7dc0ed51e0ac7bb as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/dc14de9c217746aba7dc0ed51e0ac7bb 2024-11-28T03:42:46,041 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/B of eb2f0a77050be706d70e7ee7532fcde6 into dc14de9c217746aba7dc0ed51e0ac7bb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:46,041 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:46,041 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/B, priority=12, startTime=1732765365604; duration=0sec 2024-11-28T03:42:46,041 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:46,041 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:B 2024-11-28T03:42:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:46,079 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288f987bb13f184328b7bca590b046114e_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288f987bb13f184328b7bca590b046114e_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:46,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/d55b5b832d9d43a083abc810d329682b, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:46,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/d55b5b832d9d43a083abc810d329682b is 175, key is test_row_0/A:col10/1732765363935/Put/seqid=0 2024-11-28T03:42:46,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742299_1475 (size=31255) 2024-11-28T03:42:46,087 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/d55b5b832d9d43a083abc810d329682b 2024-11-28T03:42:46,092 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/401bc9cd440840518fb66d8fb4f4663e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/401bc9cd440840518fb66d8fb4f4663e 2024-11-28T03:42:46,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/8761e222035d4f80a280ea3d496cc0ea is 50, key is test_row_0/B:col10/1732765363935/Put/seqid=0 2024-11-28T03:42:46,098 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/C of eb2f0a77050be706d70e7ee7532fcde6 into 401bc9cd440840518fb66d8fb4f4663e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:46,098 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:46,098 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/C, priority=12, startTime=1732765365604; duration=0sec 2024-11-28T03:42:46,098 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:46,098 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:C 2024-11-28T03:42:46,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742300_1476 (size=12301) 2024-11-28T03:42:46,109 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/8761e222035d4f80a280ea3d496cc0ea 2024-11-28T03:42:46,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/e6d103a6ea8f4ecd99a78678fd4cd167 is 50, key is test_row_0/C:col10/1732765363935/Put/seqid=0 2024-11-28T03:42:46,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742301_1477 (size=12301) 2024-11-28T03:42:46,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765426126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,142 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in eb2f0a77050be706d70e7ee7532fcde6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:46,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765426141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765426142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765426156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765426156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,165 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_0' in eb2f0a77050be706d70e7ee7532fcde6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,165 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in eb2f0a77050be706d70e7ee7532fcde6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765426266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765426266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765426477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765426477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,534 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/e6d103a6ea8f4ecd99a78678fd4cd167 2024-11-28T03:42:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/d55b5b832d9d43a083abc810d329682b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/d55b5b832d9d43a083abc810d329682b 2024-11-28T03:42:46,545 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/d55b5b832d9d43a083abc810d329682b, entries=150, sequenceid=483, filesize=30.5 K 2024-11-28T03:42:46,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/8761e222035d4f80a280ea3d496cc0ea as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8761e222035d4f80a280ea3d496cc0ea 2024-11-28T03:42:46,549 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8761e222035d4f80a280ea3d496cc0ea, entries=150, sequenceid=483, filesize=12.0 K 2024-11-28T03:42:46,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/e6d103a6ea8f4ecd99a78678fd4cd167 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e6d103a6ea8f4ecd99a78678fd4cd167 2024-11-28T03:42:46,554 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e6d103a6ea8f4ecd99a78678fd4cd167, entries=150, sequenceid=483, filesize=12.0 K 2024-11-28T03:42:46,555 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for eb2f0a77050be706d70e7ee7532fcde6 in 897ms, sequenceid=483, compaction requested=false 2024-11-28T03:42:46,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:46,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:46,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-28T03:42:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-28T03:42:46,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-28T03:42:46,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7420 sec 2024-11-28T03:42:46,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.7490 sec 2024-11-28T03:42:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:46,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T03:42:46,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:46,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:46,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:46,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:46,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:46,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:42:46,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128adeb9b3f921844ca80acf828c130a301_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765366065/Put/seqid=0 2024-11-28T03:42:46,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742302_1478 (size=14994) 2024-11-28T03:42:46,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765426830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765426831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765426946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:46,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765426957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:47,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765427166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:47,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765427177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:47,215 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:47,219 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128adeb9b3f921844ca80acf828c130a301_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128adeb9b3f921844ca80acf828c130a301_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:47,219 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/ec1c6171d6524eeeba2137b3d19b91ca, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:47,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/ec1c6171d6524eeeba2137b3d19b91ca is 175, key is test_row_0/A:col10/1732765366065/Put/seqid=0 2024-11-28T03:42:47,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742303_1479 (size=39949) 2024-11-28T03:42:47,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765427486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:47,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765427486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:47,605 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T03:42:47,625 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=523, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/ec1c6171d6524eeeba2137b3d19b91ca 2024-11-28T03:42:47,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/1d98fab46f15424391c213c7b9b0af3c is 50, key is test_row_0/B:col10/1732765366065/Put/seqid=0 2024-11-28T03:42:47,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742304_1480 (size=12301) 2024-11-28T03:42:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T03:42:47,967 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:47,968 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-28T03:42:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T03:42:47,969 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:47,970 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:47,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48244 deadline: 1732765428006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48330 deadline: 1732765428006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:48,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/1d98fab46f15424391c213c7b9b0af3c 2024-11-28T03:42:48,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/1130ef3ae0a8424cb1211f9d8a76c8a9 is 50, key is test_row_0/C:col10/1732765366065/Put/seqid=0 2024-11-28T03:42:48,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742305_1481 (size=12301) 2024-11-28T03:42:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T03:42:48,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-28T03:42:48,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:48,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48264 deadline: 1732765428166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:48,172 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in eb2f0a77050be706d70e7ee7532fcde6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:48,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48280 deadline: 1732765428186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:48,196 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in eb2f0a77050be706d70e7ee7532fcde6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:48,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48256 deadline: 1732765428196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:48,202 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_0' in eb2f0a77050be706d70e7ee7532fcde6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:42:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:42:48,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-28T03:42:48,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:48,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:48,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T03:42:48,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-28T03:42:48,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:48,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:48,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:48,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/1130ef3ae0a8424cb1211f9d8a76c8a9 2024-11-28T03:42:48,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/ec1c6171d6524eeeba2137b3d19b91ca as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec1c6171d6524eeeba2137b3d19b91ca 2024-11-28T03:42:48,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec1c6171d6524eeeba2137b3d19b91ca, entries=200, sequenceid=523, filesize=39.0 K 2024-11-28T03:42:48,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/1d98fab46f15424391c213c7b9b0af3c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1d98fab46f15424391c213c7b9b0af3c 2024-11-28T03:42:48,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1d98fab46f15424391c213c7b9b0af3c, entries=150, sequenceid=523, filesize=12.0 K 2024-11-28T03:42:48,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/1130ef3ae0a8424cb1211f9d8a76c8a9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1130ef3ae0a8424cb1211f9d8a76c8a9 2024-11-28T03:42:48,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1130ef3ae0a8424cb1211f9d8a76c8a9, entries=150, sequenceid=523, filesize=12.0 K 2024-11-28T03:42:48,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for eb2f0a77050be706d70e7ee7532fcde6 in 1670ms, sequenceid=523, compaction requested=true 2024-11-28T03:42:48,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:48,471 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:48,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:48,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:48,472 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:48,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:48,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:48,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb2f0a77050be706d70e7ee7532fcde6:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:48,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:48,473 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:48,473 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/A is initiating minor compaction (all files) 2024-11-28T03:42:48,473 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/A in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,473 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/960c51088a5b421d98c509afb23a6dac, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/d55b5b832d9d43a083abc810d329682b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec1c6171d6524eeeba2137b3d19b91ca] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=100.8 K 2024-11-28T03:42:48,473 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,473 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/960c51088a5b421d98c509afb23a6dac, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/d55b5b832d9d43a083abc810d329682b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec1c6171d6524eeeba2137b3d19b91ca] 2024-11-28T03:42:48,474 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:48,474 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/B is initiating minor compaction (all files) 2024-11-28T03:42:48,474 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/B in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,474 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/dc14de9c217746aba7dc0ed51e0ac7bb, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8761e222035d4f80a280ea3d496cc0ea, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1d98fab46f15424391c213c7b9b0af3c] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=36.8 K 2024-11-28T03:42:48,474 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 960c51088a5b421d98c509afb23a6dac, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732765363599 2024-11-28T03:42:48,474 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting dc14de9c217746aba7dc0ed51e0ac7bb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732765363599 2024-11-28T03:42:48,474 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d55b5b832d9d43a083abc810d329682b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732765363931 2024-11-28T03:42:48,475 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8761e222035d4f80a280ea3d496cc0ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732765363931 2024-11-28T03:42:48,475 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec1c6171d6524eeeba2137b3d19b91ca, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732765366065 2024-11-28T03:42:48,475 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d98fab46f15424391c213c7b9b0af3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732765366065 2024-11-28T03:42:48,502 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:48,502 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:48,503 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/1dfb11c681db437a826f8a9179fe58ea is 50, key is test_row_0/B:col10/1732765366065/Put/seqid=0 2024-11-28T03:42:48,509 INFO [Thread-1715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1715. 2024-11-28T03:42:48,509 DEBUG [Thread-1715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:48,509 DEBUG [Thread-1715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:48,509 DEBUG [Thread-1715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:48,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:48,509 INFO [Thread-1717 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1717. 2024-11-28T03:42:48,509 DEBUG [Thread-1717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:48,509 DEBUG [Thread-1717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:48,510 DEBUG [Thread-1717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:48,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:48,512 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411288201fb11e88b4326aec0454ca6f175d7_eb2f0a77050be706d70e7ee7532fcde6 store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:48,514 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411288201fb11e88b4326aec0454ca6f175d7_eb2f0a77050be706d70e7ee7532fcde6, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:48,515 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288201fb11e88b4326aec0454ca6f175d7_eb2f0a77050be706d70e7ee7532fcde6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:48,515 INFO [Thread-1711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1711. 2024-11-28T03:42:48,515 DEBUG [Thread-1711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:48,515 DEBUG [Thread-1711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:48,515 DEBUG [Thread-1711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:48,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:48,516 INFO [Thread-1713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1713. 2024-11-28T03:42:48,516 DEBUG [Thread-1713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:48,516 DEBUG [Thread-1713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:48,516 DEBUG [Thread-1713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:48,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:48,517 INFO [Thread-1709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1709. 2024-11-28T03:42:48,517 DEBUG [Thread-1709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:48,517 DEBUG [Thread-1709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:48,517 DEBUG [Thread-1709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:48,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:48,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742306_1482 (size=13187) 2024-11-28T03:42:48,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742307_1483 (size=4469) 2024-11-28T03:42:48,532 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/1dfb11c681db437a826f8a9179fe58ea as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1dfb11c681db437a826f8a9179fe58ea 2024-11-28T03:42:48,536 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/B of eb2f0a77050be706d70e7ee7532fcde6 into 1dfb11c681db437a826f8a9179fe58ea(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:48,536 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:48,536 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/B, priority=13, startTime=1732765368472; duration=0sec 2024-11-28T03:42:48,536 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:48,536 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:B 2024-11-28T03:42:48,536 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:48,537 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:48,537 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): eb2f0a77050be706d70e7ee7532fcde6/C is initiating minor compaction (all files) 2024-11-28T03:42:48,537 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb2f0a77050be706d70e7ee7532fcde6/C in TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,537 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/401bc9cd440840518fb66d8fb4f4663e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e6d103a6ea8f4ecd99a78678fd4cd167, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1130ef3ae0a8424cb1211f9d8a76c8a9] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp, totalSize=36.8 K 2024-11-28T03:42:48,537 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 401bc9cd440840518fb66d8fb4f4663e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732765363599 2024-11-28T03:42:48,537 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e6d103a6ea8f4ecd99a78678fd4cd167, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732765363931 2024-11-28T03:42:48,537 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1130ef3ae0a8424cb1211f9d8a76c8a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732765366065 2024-11-28T03:42:48,543 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:48,543 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/a3ba7602b3254562ac65086409c7e7e8 is 50, key is test_row_0/C:col10/1732765366065/Put/seqid=0 2024-11-28T03:42:48,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742308_1484 (size=13187) 2024-11-28T03:42:48,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-28T03:42:48,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:48,583 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:42:48,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:48,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:48,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:48,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:48,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:48,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:48,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c371fff3c44945078d0dbd966ff85274_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765366829/Put/seqid=0 2024-11-28T03:42:48,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742309_1485 (size=12454) 2024-11-28T03:42:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T03:42:48,929 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb2f0a77050be706d70e7ee7532fcde6#A#compaction#409 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:48,930 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/c120073e239d462bb4ed6646207786fd is 175, key is test_row_0/A:col10/1732765366065/Put/seqid=0 2024-11-28T03:42:48,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742310_1486 (size=32141) 2024-11-28T03:42:48,950 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/a3ba7602b3254562ac65086409c7e7e8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/a3ba7602b3254562ac65086409c7e7e8 2024-11-28T03:42:48,954 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/C of eb2f0a77050be706d70e7ee7532fcde6 into a3ba7602b3254562ac65086409c7e7e8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:48,954 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:48,954 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/C, priority=13, startTime=1732765368472; duration=0sec 2024-11-28T03:42:48,954 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:48,954 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:C 2024-11-28T03:42:48,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:48,994 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c371fff3c44945078d0dbd966ff85274_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c371fff3c44945078d0dbd966ff85274_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:48,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/361b6118afad4e2780da3711c3d4110f, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:48,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/361b6118afad4e2780da3711c3d4110f is 175, key is test_row_0/A:col10/1732765366829/Put/seqid=0 2024-11-28T03:42:48,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742311_1487 (size=31255) 2024-11-28T03:42:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:49,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. as already flushing 2024-11-28T03:42:49,028 INFO [Thread-1698 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1698. 2024-11-28T03:42:49,028 INFO [Thread-1704 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1704. 2024-11-28T03:42:49,028 DEBUG [Thread-1698 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:49,028 DEBUG [Thread-1704 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:49,028 DEBUG [Thread-1698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:49,028 DEBUG [Thread-1704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:49,028 DEBUG [Thread-1698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:49,028 DEBUG [Thread-1704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:49,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:49,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T03:42:49,338 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/c120073e239d462bb4ed6646207786fd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/c120073e239d462bb4ed6646207786fd 2024-11-28T03:42:49,342 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb2f0a77050be706d70e7ee7532fcde6/A of eb2f0a77050be706d70e7ee7532fcde6 into c120073e239d462bb4ed6646207786fd(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:49,342 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:49,342 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., storeName=eb2f0a77050be706d70e7ee7532fcde6/A, priority=13, startTime=1732765368471; duration=0sec 2024-11-28T03:42:49,342 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:49,342 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb2f0a77050be706d70e7ee7532fcde6:A 2024-11-28T03:42:49,399 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=534, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/361b6118afad4e2780da3711c3d4110f 2024-11-28T03:42:49,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/39683a62246a4fa88a263d460e8d7a57 is 50, key is test_row_0/B:col10/1732765366829/Put/seqid=0 2024-11-28T03:42:49,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742312_1488 (size=12301) 2024-11-28T03:42:49,808 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/39683a62246a4fa88a263d460e8d7a57 2024-11-28T03:42:49,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/06170b7cd36d46f48ca4980e9e17c5f1 is 50, key is test_row_0/C:col10/1732765366829/Put/seqid=0 2024-11-28T03:42:49,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742313_1489 (size=12301) 2024-11-28T03:42:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T03:42:50,219 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/06170b7cd36d46f48ca4980e9e17c5f1 2024-11-28T03:42:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/361b6118afad4e2780da3711c3d4110f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/361b6118afad4e2780da3711c3d4110f 2024-11-28T03:42:50,227 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/361b6118afad4e2780da3711c3d4110f, entries=150, sequenceid=534, filesize=30.5 K 2024-11-28T03:42:50,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/39683a62246a4fa88a263d460e8d7a57 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/39683a62246a4fa88a263d460e8d7a57 2024-11-28T03:42:50,231 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/39683a62246a4fa88a263d460e8d7a57, entries=150, sequenceid=534, filesize=12.0 K 2024-11-28T03:42:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/06170b7cd36d46f48ca4980e9e17c5f1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/06170b7cd36d46f48ca4980e9e17c5f1 2024-11-28T03:42:50,234 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/06170b7cd36d46f48ca4980e9e17c5f1, entries=150, sequenceid=534, filesize=12.0 K 2024-11-28T03:42:50,235 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for eb2f0a77050be706d70e7ee7532fcde6 in 1653ms, sequenceid=534, compaction requested=false 2024-11-28T03:42:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for eb2f0a77050be706d70e7ee7532fcde6: 2024-11-28T03:42:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-28T03:42:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-28T03:42:50,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-28T03:42:50,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2670 sec 2024-11-28T03:42:50,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.2710 sec 2024-11-28T03:42:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-28T03:42:52,136 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:52,198 INFO [Thread-1700 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1700. 2024-11-28T03:42:52,198 DEBUG [Thread-1700 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:52,198 DEBUG [Thread-1700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:52,198 DEBUG [Thread-1700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:52,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:52,217 INFO [Thread-1702 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1702. 2024-11-28T03:42:52,217 DEBUG [Thread-1702 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:52,217 DEBUG [Thread-1702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:52,217 DEBUG [Thread-1702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:52,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:52,247 INFO [Thread-1706 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1706. 2024-11-28T03:42:52,247 DEBUG [Thread-1706 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:42:52,247 DEBUG [Thread-1706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:52,248 DEBUG [Thread-1706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:52,248 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 878 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2629 rows 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 877 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2628 rows 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 878 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2633 rows 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 878 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2633 rows 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 877 2024-11-28T03:42:52,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2630 rows 2024-11-28T03:42:52,248 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:42:52,248 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:42:52,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:52,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:52,248 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:42:52,249 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T03:42:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T03:42:52,253 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765372252"}]},"ts":"1732765372252"} 2024-11-28T03:42:52,254 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T03:42:52,254 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T03:42:52,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T03:42:52,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, UNASSIGN}] 2024-11-28T03:42:52,257 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, UNASSIGN 2024-11-28T03:42:52,257 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=eb2f0a77050be706d70e7ee7532fcde6, regionState=CLOSING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:42:52,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, UNASSIGN because future has completed 2024-11-28T03:42:52,259 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T03:42:52,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:42:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T03:42:52,411 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(122): Close eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:52,411 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T03:42:52,411 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1722): Closing eb2f0a77050be706d70e7ee7532fcde6, disabling compactions & flushes 2024-11-28T03:42:52,411 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:52,411 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:52,411 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. after waiting 0 ms 2024-11-28T03:42:52,411 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:52,411 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(2902): Flushing eb2f0a77050be706d70e7ee7532fcde6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T03:42:52,412 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=A 2024-11-28T03:42:52,412 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:52,412 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=B 2024-11-28T03:42:52,412 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:52,412 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eb2f0a77050be706d70e7ee7532fcde6, store=C 2024-11-28T03:42:52,412 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:52,417 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128dc12a80dff9244bbab7b0efb05637493_eb2f0a77050be706d70e7ee7532fcde6 is 50, key is test_row_0/A:col10/1732765372246/Put/seqid=0 2024-11-28T03:42:52,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742314_1490 (size=12454) 2024-11-28T03:42:52,422 WARN [regionserver/3a546af21488:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-28T03:42:52,424 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/cd14a988d1d64c45b71820f5db31bb11, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/c4dd731ec9a24ac7bbaf17c536c16be4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/6f6b4445a65e440199d1c726037954d5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3a047cce000749f7a44c0f98824c53c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3dcef47bf4c84b93a873e031615ac585, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/fd5307964d3449baa2efc7c38c421882, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/42f4f9dd60e24f35b69671fd972465c2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec5abbca4e224ab58a4060279649e2d0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/efb6f11b37b841cdb71831d3ffcf984e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/566077488d6f4952b797b1a7eaa6431d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/a7958cc12c344c08ba083e240439c040, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/874edc99185b4468bd2ff1bf1e1d107f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/046afd8fe5b549d8aca75818882cf639, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/f16d04538a934f90839c429b6b3491dd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/357947f0b7ba40bfbd7ea86947f406c5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/497314f0c795400c969a8be2bba64d3c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/da18c15cc9c44bf08657df218b153204, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/b1592af444fd4dfab2ba3d3737555099, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7ab8a82dfc644f57b740d3ae619df2bf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/9108614a124143c9a7e962206259e387, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/e8d296accc3b4c128436d8f4eb82d8a8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/db00480f4f9c41e3bd9a6140cb42b1ea, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/960c51088a5b421d98c509afb23a6dac, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7cdd721b78bc45229d382b224c99c577, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/d55b5b832d9d43a083abc810d329682b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec1c6171d6524eeeba2137b3d19b91ca] to archive 2024-11-28T03:42:52,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:42:52,426 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/cd14a988d1d64c45b71820f5db31bb11 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/cd14a988d1d64c45b71820f5db31bb11 2024-11-28T03:42:52,427 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/c4dd731ec9a24ac7bbaf17c536c16be4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/c4dd731ec9a24ac7bbaf17c536c16be4 2024-11-28T03:42:52,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/6f6b4445a65e440199d1c726037954d5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/6f6b4445a65e440199d1c726037954d5 2024-11-28T03:42:52,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3a047cce000749f7a44c0f98824c53c4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3a047cce000749f7a44c0f98824c53c4 2024-11-28T03:42:52,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3dcef47bf4c84b93a873e031615ac585 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/3dcef47bf4c84b93a873e031615ac585 2024-11-28T03:42:52,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/fd5307964d3449baa2efc7c38c421882 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/fd5307964d3449baa2efc7c38c421882 2024-11-28T03:42:52,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/42f4f9dd60e24f35b69671fd972465c2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/42f4f9dd60e24f35b69671fd972465c2 2024-11-28T03:42:52,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec5abbca4e224ab58a4060279649e2d0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec5abbca4e224ab58a4060279649e2d0 2024-11-28T03:42:52,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/efb6f11b37b841cdb71831d3ffcf984e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/efb6f11b37b841cdb71831d3ffcf984e 2024-11-28T03:42:52,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/566077488d6f4952b797b1a7eaa6431d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/566077488d6f4952b797b1a7eaa6431d 2024-11-28T03:42:52,434 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/a7958cc12c344c08ba083e240439c040 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/a7958cc12c344c08ba083e240439c040 2024-11-28T03:42:52,434 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/874edc99185b4468bd2ff1bf1e1d107f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/874edc99185b4468bd2ff1bf1e1d107f 2024-11-28T03:42:52,435 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/046afd8fe5b549d8aca75818882cf639 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/046afd8fe5b549d8aca75818882cf639 2024-11-28T03:42:52,436 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/f16d04538a934f90839c429b6b3491dd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/f16d04538a934f90839c429b6b3491dd 2024-11-28T03:42:52,437 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/357947f0b7ba40bfbd7ea86947f406c5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/357947f0b7ba40bfbd7ea86947f406c5 2024-11-28T03:42:52,438 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/497314f0c795400c969a8be2bba64d3c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/497314f0c795400c969a8be2bba64d3c 2024-11-28T03:42:52,438 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/da18c15cc9c44bf08657df218b153204 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/da18c15cc9c44bf08657df218b153204 2024-11-28T03:42:52,439 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/b1592af444fd4dfab2ba3d3737555099 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/b1592af444fd4dfab2ba3d3737555099 2024-11-28T03:42:52,440 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7ab8a82dfc644f57b740d3ae619df2bf to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7ab8a82dfc644f57b740d3ae619df2bf 2024-11-28T03:42:52,441 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/9108614a124143c9a7e962206259e387 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/9108614a124143c9a7e962206259e387 2024-11-28T03:42:52,441 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/e8d296accc3b4c128436d8f4eb82d8a8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/e8d296accc3b4c128436d8f4eb82d8a8 2024-11-28T03:42:52,442 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/db00480f4f9c41e3bd9a6140cb42b1ea to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/db00480f4f9c41e3bd9a6140cb42b1ea 2024-11-28T03:42:52,443 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/960c51088a5b421d98c509afb23a6dac to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/960c51088a5b421d98c509afb23a6dac 2024-11-28T03:42:52,444 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7cdd721b78bc45229d382b224c99c577 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/7cdd721b78bc45229d382b224c99c577 2024-11-28T03:42:52,444 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/d55b5b832d9d43a083abc810d329682b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/d55b5b832d9d43a083abc810d329682b 2024-11-28T03:42:52,445 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec1c6171d6524eeeba2137b3d19b91ca to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/ec1c6171d6524eeeba2137b3d19b91ca 2024-11-28T03:42:52,448 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/ced927c2b4e348c5a1de06556dc09e07, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/d6574371572b4d958e4e752ce78765a5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/584f28947c1145b299d0e48073f5bf39, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4153e35401c3453e8fe28dd82c145b5a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/5f88b3deff004e779183efd0e4945d1c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/347167cdfbd94caa9d37dde2c98c9edd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e5b2e27d4c32407fa59051410e67c31d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/92cb87602b2c4060977086962dce9b7d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/740d473820d74bbd94fd8fbaafde276b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b5823df254ca460ca4a9412b9d7ed74a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e7a3a64cb9f5473a804a24901dc75075, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/f3452f58f0534aff9db8291718e4e563, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8d0bede7f0114fdda18966a112f164bc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/0ca59e147f6946dd9ee4efd26abeaaca, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/55605e52ad124fbaba01140c03a683c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4d0660f66e8f4801a0c5b390d5487680, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1bfc6610e64a4052bf8730c967f594bd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/11b80f7316db450db3b8a3e50399d469, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/63dbf37bd4ec4c15a3b9dc07707dee8e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8f5ca5d0489f425da23db0b93305caae, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/6d355f32c9034291bfa9669921c788c2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b34fe0ea3492411d83a0e19daa2d7e08, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/dc14de9c217746aba7dc0ed51e0ac7bb, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/fcb44397dc88461d84f40d30ed9c127e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8761e222035d4f80a280ea3d496cc0ea, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1d98fab46f15424391c213c7b9b0af3c] to archive 2024-11-28T03:42:52,449 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:42:52,450 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/ced927c2b4e348c5a1de06556dc09e07 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/ced927c2b4e348c5a1de06556dc09e07 2024-11-28T03:42:52,450 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/d6574371572b4d958e4e752ce78765a5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/d6574371572b4d958e4e752ce78765a5 2024-11-28T03:42:52,451 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/584f28947c1145b299d0e48073f5bf39 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/584f28947c1145b299d0e48073f5bf39 2024-11-28T03:42:52,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4153e35401c3453e8fe28dd82c145b5a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4153e35401c3453e8fe28dd82c145b5a 2024-11-28T03:42:52,453 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/5f88b3deff004e779183efd0e4945d1c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/5f88b3deff004e779183efd0e4945d1c 2024-11-28T03:42:52,454 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/347167cdfbd94caa9d37dde2c98c9edd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/347167cdfbd94caa9d37dde2c98c9edd 2024-11-28T03:42:52,455 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e5b2e27d4c32407fa59051410e67c31d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e5b2e27d4c32407fa59051410e67c31d 2024-11-28T03:42:52,456 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/92cb87602b2c4060977086962dce9b7d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/92cb87602b2c4060977086962dce9b7d 2024-11-28T03:42:52,457 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/740d473820d74bbd94fd8fbaafde276b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/740d473820d74bbd94fd8fbaafde276b 2024-11-28T03:42:52,457 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b5823df254ca460ca4a9412b9d7ed74a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b5823df254ca460ca4a9412b9d7ed74a 2024-11-28T03:42:52,458 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e7a3a64cb9f5473a804a24901dc75075 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/e7a3a64cb9f5473a804a24901dc75075 2024-11-28T03:42:52,459 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/f3452f58f0534aff9db8291718e4e563 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/f3452f58f0534aff9db8291718e4e563 2024-11-28T03:42:52,459 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8d0bede7f0114fdda18966a112f164bc to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8d0bede7f0114fdda18966a112f164bc 2024-11-28T03:42:52,460 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/0ca59e147f6946dd9ee4efd26abeaaca to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/0ca59e147f6946dd9ee4efd26abeaaca 2024-11-28T03:42:52,461 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/55605e52ad124fbaba01140c03a683c4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/55605e52ad124fbaba01140c03a683c4 2024-11-28T03:42:52,462 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4d0660f66e8f4801a0c5b390d5487680 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/4d0660f66e8f4801a0c5b390d5487680 2024-11-28T03:42:52,462 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1bfc6610e64a4052bf8730c967f594bd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1bfc6610e64a4052bf8730c967f594bd 2024-11-28T03:42:52,463 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/11b80f7316db450db3b8a3e50399d469 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/11b80f7316db450db3b8a3e50399d469 2024-11-28T03:42:52,464 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/63dbf37bd4ec4c15a3b9dc07707dee8e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/63dbf37bd4ec4c15a3b9dc07707dee8e 2024-11-28T03:42:52,465 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8f5ca5d0489f425da23db0b93305caae to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8f5ca5d0489f425da23db0b93305caae 2024-11-28T03:42:52,465 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/6d355f32c9034291bfa9669921c788c2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/6d355f32c9034291bfa9669921c788c2 2024-11-28T03:42:52,466 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b34fe0ea3492411d83a0e19daa2d7e08 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b34fe0ea3492411d83a0e19daa2d7e08 2024-11-28T03:42:52,467 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/dc14de9c217746aba7dc0ed51e0ac7bb to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/dc14de9c217746aba7dc0ed51e0ac7bb 2024-11-28T03:42:52,468 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/fcb44397dc88461d84f40d30ed9c127e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/fcb44397dc88461d84f40d30ed9c127e 2024-11-28T03:42:52,468 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8761e222035d4f80a280ea3d496cc0ea to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/8761e222035d4f80a280ea3d496cc0ea 2024-11-28T03:42:52,469 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1d98fab46f15424391c213c7b9b0af3c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1d98fab46f15424391c213c7b9b0af3c 2024-11-28T03:42:52,472 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/039dc5e7ca1b4ea9a26301a2323c69fe, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e7b77600e7864d46803f5c7fcc5cc484, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d11cd6c434814444ac404d3ce96ba46c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/5f02c1a1d65042e1ad02748c6860133c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/8269f9eeb9bc49969a2b260d5e514d35, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/506c57cd658a481c9207eb3dc357de3c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/36894d639edc4c869675eb859397eb1b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/519f3fa6fa9e49ff8a111ae94d318626, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7e29ea7fcd13449ebab4084be8c89dff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/b3dc511c2cb54750aa374ebcf01b9b24, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/81e678bb47374f32a2a0aa1b524562bf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9de559bd05434cf1989f0b5abb96a9c3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/c5a62438a2fb4d11a98c6411f3817219, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/65fd94dd0d4d42abb82bbbea004c7bf9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7fe633c04c2846f1bb1cd9b4c9d0a78f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/a301d35aa1264fcda47c33c5db95f24c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1b04cc7397174977b2a87a4d7f85d006, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/016a4a26ad3444b6a7fd4f7337634ac3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/36dfae2124a04b45a88b35825988128c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/83343608da464e1aa79a02a201f847ae, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9f2b06b19a7c43948cc34587e04c6099, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/11559eb8ed3e4b25853199a17895990a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/401bc9cd440840518fb66d8fb4f4663e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d56ce54268714712915cf0232566488c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e6d103a6ea8f4ecd99a78678fd4cd167, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1130ef3ae0a8424cb1211f9d8a76c8a9] to archive 2024-11-28T03:42:52,473 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:42:52,474 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/039dc5e7ca1b4ea9a26301a2323c69fe to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/039dc5e7ca1b4ea9a26301a2323c69fe 2024-11-28T03:42:52,475 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e7b77600e7864d46803f5c7fcc5cc484 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e7b77600e7864d46803f5c7fcc5cc484 2024-11-28T03:42:52,475 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d11cd6c434814444ac404d3ce96ba46c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d11cd6c434814444ac404d3ce96ba46c 2024-11-28T03:42:52,476 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/5f02c1a1d65042e1ad02748c6860133c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/5f02c1a1d65042e1ad02748c6860133c 2024-11-28T03:42:52,477 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/8269f9eeb9bc49969a2b260d5e514d35 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/8269f9eeb9bc49969a2b260d5e514d35 2024-11-28T03:42:52,477 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/506c57cd658a481c9207eb3dc357de3c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/506c57cd658a481c9207eb3dc357de3c 2024-11-28T03:42:52,478 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/36894d639edc4c869675eb859397eb1b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/36894d639edc4c869675eb859397eb1b 2024-11-28T03:42:52,479 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/519f3fa6fa9e49ff8a111ae94d318626 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/519f3fa6fa9e49ff8a111ae94d318626 2024-11-28T03:42:52,480 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7e29ea7fcd13449ebab4084be8c89dff to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7e29ea7fcd13449ebab4084be8c89dff 2024-11-28T03:42:52,480 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/b3dc511c2cb54750aa374ebcf01b9b24 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/b3dc511c2cb54750aa374ebcf01b9b24 2024-11-28T03:42:52,481 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/81e678bb47374f32a2a0aa1b524562bf to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/81e678bb47374f32a2a0aa1b524562bf 2024-11-28T03:42:52,482 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9de559bd05434cf1989f0b5abb96a9c3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9de559bd05434cf1989f0b5abb96a9c3 2024-11-28T03:42:52,483 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/c5a62438a2fb4d11a98c6411f3817219 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/c5a62438a2fb4d11a98c6411f3817219 2024-11-28T03:42:52,483 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/65fd94dd0d4d42abb82bbbea004c7bf9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/65fd94dd0d4d42abb82bbbea004c7bf9 2024-11-28T03:42:52,484 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7fe633c04c2846f1bb1cd9b4c9d0a78f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/7fe633c04c2846f1bb1cd9b4c9d0a78f 2024-11-28T03:42:52,485 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/a301d35aa1264fcda47c33c5db95f24c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/a301d35aa1264fcda47c33c5db95f24c 2024-11-28T03:42:52,486 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1b04cc7397174977b2a87a4d7f85d006 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1b04cc7397174977b2a87a4d7f85d006 2024-11-28T03:42:52,486 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/016a4a26ad3444b6a7fd4f7337634ac3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/016a4a26ad3444b6a7fd4f7337634ac3 2024-11-28T03:42:52,487 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/36dfae2124a04b45a88b35825988128c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/36dfae2124a04b45a88b35825988128c 2024-11-28T03:42:52,488 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/83343608da464e1aa79a02a201f847ae to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/83343608da464e1aa79a02a201f847ae 2024-11-28T03:42:52,489 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9f2b06b19a7c43948cc34587e04c6099 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/9f2b06b19a7c43948cc34587e04c6099 2024-11-28T03:42:52,490 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/11559eb8ed3e4b25853199a17895990a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/11559eb8ed3e4b25853199a17895990a 2024-11-28T03:42:52,490 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/401bc9cd440840518fb66d8fb4f4663e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/401bc9cd440840518fb66d8fb4f4663e 2024-11-28T03:42:52,491 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d56ce54268714712915cf0232566488c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/d56ce54268714712915cf0232566488c 2024-11-28T03:42:52,492 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e6d103a6ea8f4ecd99a78678fd4cd167 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/e6d103a6ea8f4ecd99a78678fd4cd167 2024-11-28T03:42:52,492 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1130ef3ae0a8424cb1211f9d8a76c8a9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/1130ef3ae0a8424cb1211f9d8a76c8a9 2024-11-28T03:42:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T03:42:52,821 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:52,824 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128dc12a80dff9244bbab7b0efb05637493_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dc12a80dff9244bbab7b0efb05637493_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:52,825 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/02b89428f66b4858884a8170f83d8359, store: [table=TestAcidGuarantees family=A region=eb2f0a77050be706d70e7ee7532fcde6] 2024-11-28T03:42:52,825 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/02b89428f66b4858884a8170f83d8359 is 175, key is test_row_0/A:col10/1732765372246/Put/seqid=0 2024-11-28T03:42:52,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742315_1491 (size=31255) 2024-11-28T03:42:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T03:42:53,229 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=544, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/02b89428f66b4858884a8170f83d8359 2024-11-28T03:42:53,235 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/b844924e522442598f2bfd215014f93f is 50, key is test_row_0/B:col10/1732765372246/Put/seqid=0 2024-11-28T03:42:53,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742316_1492 (size=12301) 2024-11-28T03:42:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T03:42:53,639 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/b844924e522442598f2bfd215014f93f 2024-11-28T03:42:53,644 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/6896329893b844e9ae3558c82687ba76 is 50, key is test_row_0/C:col10/1732765372246/Put/seqid=0 2024-11-28T03:42:53,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742317_1493 (size=12301) 2024-11-28T03:42:54,048 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/6896329893b844e9ae3558c82687ba76 2024-11-28T03:42:54,052 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/A/02b89428f66b4858884a8170f83d8359 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/02b89428f66b4858884a8170f83d8359 2024-11-28T03:42:54,055 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/02b89428f66b4858884a8170f83d8359, entries=150, sequenceid=544, filesize=30.5 K 2024-11-28T03:42:54,055 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/B/b844924e522442598f2bfd215014f93f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b844924e522442598f2bfd215014f93f 2024-11-28T03:42:54,058 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b844924e522442598f2bfd215014f93f, entries=150, sequenceid=544, filesize=12.0 K 2024-11-28T03:42:54,058 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/.tmp/C/6896329893b844e9ae3558c82687ba76 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/6896329893b844e9ae3558c82687ba76 2024-11-28T03:42:54,061 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/6896329893b844e9ae3558c82687ba76, entries=150, sequenceid=544, filesize=12.0 K 2024-11-28T03:42:54,062 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for eb2f0a77050be706d70e7ee7532fcde6 in 1651ms, sequenceid=544, compaction requested=true 2024-11-28T03:42:54,066 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/recovered.edits/547.seqid, newMaxSeqId=547, maxSeqId=4 2024-11-28T03:42:54,066 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. 2024-11-28T03:42:54,067 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1676): Region close journal for eb2f0a77050be706d70e7ee7532fcde6: Waiting for close lock at 1732765372411Running coprocessor pre-close hooks at 1732765372411Disabling compacts and flushes for region at 1732765372411Disabling writes for close at 1732765372411Obtaining lock to block concurrent updates at 1732765372411Preparing flush snapshotting stores in eb2f0a77050be706d70e7ee7532fcde6 at 1732765372411Finished memstore snapshotting TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732765372412 (+1 ms)Flushing stores of TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6. at 1732765372412Flushing eb2f0a77050be706d70e7ee7532fcde6/A: creating writer at 1732765372412Flushing mob file eb2f0a77050be706d70e7ee7532fcde6/A: appending metadata at 1732765372417 (+5 ms)Flushing mob file eb2f0a77050be706d70e7ee7532fcde6/A: closing flushed file at 1732765372417Flushing eb2f0a77050be706d70e7ee7532fcde6/A: appending metadata at 1732765372825 (+408 ms)Flushing eb2f0a77050be706d70e7ee7532fcde6/A: closing flushed file at 1732765372825Flushing eb2f0a77050be706d70e7ee7532fcde6/B: creating writer at 1732765373232 (+407 ms)Flushing eb2f0a77050be706d70e7ee7532fcde6/B: appending metadata at 1732765373235 (+3 ms)Flushing eb2f0a77050be706d70e7ee7532fcde6/B: closing flushed file at 1732765373235Flushing eb2f0a77050be706d70e7ee7532fcde6/C: creating writer at 1732765373642 (+407 ms)Flushing eb2f0a77050be706d70e7ee7532fcde6/C: appending metadata at 1732765373644 (+2 ms)Flushing eb2f0a77050be706d70e7ee7532fcde6/C: closing flushed file at 1732765373644Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2804a478: reopening flushed file at 1732765374051 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@204def41: reopening flushed file at 1732765374055 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a000977: reopening flushed file at 1732765374058 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for eb2f0a77050be706d70e7ee7532fcde6 in 1651ms, sequenceid=544, compaction requested=true at 1732765374062 (+4 ms)Writing region close event to WAL at 1732765374063 (+1 ms)Running coprocessor post-close hooks at 1732765374066 (+3 ms)Closed at 1732765374066 2024-11-28T03:42:54,068 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(157): Closed eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,069 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=eb2f0a77050be706d70e7ee7532fcde6, regionState=CLOSED 2024-11-28T03:42:54,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:42:54,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-28T03:42:54,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseRegionProcedure eb2f0a77050be706d70e7ee7532fcde6, server=3a546af21488,34671,1732765249447 in 1.8120 sec 2024-11-28T03:42:54,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-28T03:42:54,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb2f0a77050be706d70e7ee7532fcde6, UNASSIGN in 1.8160 sec 2024-11-28T03:42:54,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-28T03:42:54,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8190 sec 2024-11-28T03:42:54,076 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765374076"}]},"ts":"1732765374076"} 2024-11-28T03:42:54,077 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T03:42:54,078 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T03:42:54,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8290 sec 2024-11-28T03:42:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T03:42:54,396 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:54,397 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T03:42:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:54,399 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T03:42:54,400 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=125, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:54,401 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,403 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/recovered.edits] 2024-11-28T03:42:54,405 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/02b89428f66b4858884a8170f83d8359 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/02b89428f66b4858884a8170f83d8359 2024-11-28T03:42:54,406 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/361b6118afad4e2780da3711c3d4110f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/361b6118afad4e2780da3711c3d4110f 2024-11-28T03:42:54,407 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/c120073e239d462bb4ed6646207786fd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/A/c120073e239d462bb4ed6646207786fd 2024-11-28T03:42:54,409 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1dfb11c681db437a826f8a9179fe58ea to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/1dfb11c681db437a826f8a9179fe58ea 2024-11-28T03:42:54,409 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/39683a62246a4fa88a263d460e8d7a57 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/39683a62246a4fa88a263d460e8d7a57 2024-11-28T03:42:54,410 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b844924e522442598f2bfd215014f93f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/B/b844924e522442598f2bfd215014f93f 2024-11-28T03:42:54,412 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/06170b7cd36d46f48ca4980e9e17c5f1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/06170b7cd36d46f48ca4980e9e17c5f1 2024-11-28T03:42:54,413 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/6896329893b844e9ae3558c82687ba76 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/6896329893b844e9ae3558c82687ba76 2024-11-28T03:42:54,414 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/a3ba7602b3254562ac65086409c7e7e8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/C/a3ba7602b3254562ac65086409c7e7e8 2024-11-28T03:42:54,416 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/recovered.edits/547.seqid to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6/recovered.edits/547.seqid 2024-11-28T03:42:54,416 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,416 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T03:42:54,417 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T03:42:54,417 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T03:42:54,419 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112808137823404747e9958dbf2c5dffb814_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112808137823404747e9958dbf2c5dffb814_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,420 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128188da920d5e64f11b913c1e413c77f51_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128188da920d5e64f11b913c1e413c77f51_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,421 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281cf45cadab6a4969a05e6079490e6e98_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281cf45cadab6a4969a05e6079490e6e98_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,422 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284c69ae651d5d4471918bd5ebb0e80f7a_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284c69ae651d5d4471918bd5ebb0e80f7a_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,423 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284e16843ffec74478be15f4ab253c40d8_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284e16843ffec74478be15f4ab253c40d8_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,424 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112863575a7893d3413c84e0831a16ade87e_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112863575a7893d3413c84e0831a16ade87e_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,425 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128636cea58c53a450580a99c425b334f51_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128636cea58c53a450580a99c425b334f51_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,426 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112882ceda026bf9486aa5497f133641e802_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112882ceda026bf9486aa5497f133641e802_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,427 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128863c055f0d7f48c4bd671eea8dc9026f_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128863c055f0d7f48c4bd671eea8dc9026f_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,428 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288acfa0ffc95e4e6cbe63722a8973e6d3_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288acfa0ffc95e4e6cbe63722a8973e6d3_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,429 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288f987bb13f184328b7bca590b046114e_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288f987bb13f184328b7bca590b046114e_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,429 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128adeb9b3f921844ca80acf828c130a301_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128adeb9b3f921844ca80acf828c130a301_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,430 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c2ebf0e5dfbf4b17bb021f51b3b5a136_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c2ebf0e5dfbf4b17bb021f51b3b5a136_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,431 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c371fff3c44945078d0dbd966ff85274_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c371fff3c44945078d0dbd966ff85274_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,432 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c4d02a79f5044b7d9337b4769d23c30b_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c4d02a79f5044b7d9337b4769d23c30b_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,433 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d10a9e51ff244a5982db188a219c6b2c_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d10a9e51ff244a5982db188a219c6b2c_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,434 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d18b6f5b3f364ebc981df3e5331b7154_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d18b6f5b3f364ebc981df3e5331b7154_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,435 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d7fe867076d44186839c4e81d9e721b3_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d7fe867076d44186839c4e81d9e721b3_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,435 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dc12a80dff9244bbab7b0efb05637493_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dc12a80dff9244bbab7b0efb05637493_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,436 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e447be7aa85d4c0d827238c477c58b38_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e447be7aa85d4c0d827238c477c58b38_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,437 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f6ab0761e15d44999fbde0f0f6ce2588_eb2f0a77050be706d70e7ee7532fcde6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f6ab0761e15d44999fbde0f0f6ce2588_eb2f0a77050be706d70e7ee7532fcde6 2024-11-28T03:42:54,438 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T03:42:54,439 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=125, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:54,441 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T03:42:54,443 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T03:42:54,444 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=125, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:54,444 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T03:42:54,444 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732765374444"}]},"ts":"9223372036854775807"} 2024-11-28T03:42:54,446 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T03:42:54,446 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => eb2f0a77050be706d70e7ee7532fcde6, NAME => 'TestAcidGuarantees,,1732765345390.eb2f0a77050be706d70e7ee7532fcde6.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T03:42:54,446 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T03:42:54,446 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732765374446"}]},"ts":"9223372036854775807"} 2024-11-28T03:42:54,447 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T03:42:54,448 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=125, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:54,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-28T03:42:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T03:42:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T03:42:54,506 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:54,518 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 250) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_764232134_22 at /127.0.0.1:52586 [Waiting for operation #1092] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-769855082_22 at /127.0.0.1:51136 [Waiting for operation #721] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-769855082_22 at /127.0.0.1:51150 [Waiting for operation #699] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_764232134_22 at /127.0.0.1:51134 [Waiting for operation #705] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=478 (was 486), ProcessCount=11 (was 11), AvailableMemoryMB=6184 (was 6266) 2024-11-28T03:42:54,526 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=478, ProcessCount=11, AvailableMemoryMB=6183 2024-11-28T03:42:54,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T03:42:54,528 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:42:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T03:42:54,530 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:42:54,530 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:54,530 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 126 2024-11-28T03:42:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-28T03:42:54,531 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:42:54,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742318_1494 (size=1061) 2024-11-28T03:42:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-28T03:42:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-28T03:42:54,937 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72 2024-11-28T03:42:54,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742319_1495 (size=53) 2024-11-28T03:42:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-28T03:42:55,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:42:55,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e2c8ad458d2ec9e8bad5d996a8a6cef0, disabling compactions & flushes 2024-11-28T03:42:55,343 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:55,343 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:55,343 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. after waiting 0 ms 2024-11-28T03:42:55,343 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:55,343 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:55,343 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: Waiting for close lock at 1732765375342Disabling compacts and flushes for region at 1732765375342Disabling writes for close at 1732765375343 (+1 ms)Writing region close event to WAL at 1732765375343Closed at 1732765375343 2024-11-28T03:42:55,344 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:42:55,344 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732765375344"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732765375344"}]},"ts":"1732765375344"} 2024-11-28T03:42:55,345 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:42:55,346 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:42:55,346 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765375346"}]},"ts":"1732765375346"} 2024-11-28T03:42:55,347 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T03:42:55,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2c8ad458d2ec9e8bad5d996a8a6cef0, ASSIGN}] 2024-11-28T03:42:55,348 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2c8ad458d2ec9e8bad5d996a8a6cef0, ASSIGN 2024-11-28T03:42:55,348 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2c8ad458d2ec9e8bad5d996a8a6cef0, ASSIGN; state=OFFLINE, location=3a546af21488,34671,1732765249447; forceNewPlan=false, retain=false 2024-11-28T03:42:55,499 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=e2c8ad458d2ec9e8bad5d996a8a6cef0, regionState=OPENING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:42:55,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2c8ad458d2ec9e8bad5d996a8a6cef0, ASSIGN because future has completed 2024-11-28T03:42:55,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:42:55,654 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:55,655 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7752): Opening region: {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:42:55,656 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,656 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:42:55,656 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7794): checking encryption for e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,656 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7797): checking classloading for e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,657 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,658 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:42:55,658 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2c8ad458d2ec9e8bad5d996a8a6cef0 columnFamilyName A 2024-11-28T03:42:55,658 DEBUG [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:55,658 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] regionserver.HStore(327): Store=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:42:55,658 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,659 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:42:55,659 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2c8ad458d2ec9e8bad5d996a8a6cef0 columnFamilyName B 2024-11-28T03:42:55,659 DEBUG [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:55,660 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] regionserver.HStore(327): Store=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:42:55,660 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,660 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:42:55,661 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2c8ad458d2ec9e8bad5d996a8a6cef0 columnFamilyName C 2024-11-28T03:42:55,661 DEBUG [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:42:55,661 INFO [StoreOpener-e2c8ad458d2ec9e8bad5d996a8a6cef0-1 {}] regionserver.HStore(327): Store=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:42:55,661 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:55,661 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1038): replaying wal for e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,662 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,662 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,662 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1048): stopping wal replay for e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,663 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1060): Cleaning up temporary data for e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,663 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:42:55,664 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1093): writing seq id for e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,665 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:42:55,665 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1114): Opened e2c8ad458d2ec9e8bad5d996a8a6cef0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73205567, jitterRate=0.09084795415401459}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:42:55,665 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-28T03:42:55,666 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1006): Region open journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: Running coprocessor pre-open hook at 1732765375656Writing region info on filesystem at 1732765375656Initializing all the Stores at 1732765375656Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765375656Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765375657 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765375657Cleaning up temporary data from old regions at 1732765375663 (+6 ms)Running coprocessor post-open hooks at 1732765375665 (+2 ms)Region opened successfully at 1732765375666 (+1 ms) 2024-11-28T03:42:55,667 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., pid=128, masterSystemTime=1732765375652 2024-11-28T03:42:55,668 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:55,668 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:55,669 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=e2c8ad458d2ec9e8bad5d996a8a6cef0, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:42:55,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:42:55,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-28T03:42:55,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; OpenRegionProcedure e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 in 169 msec 2024-11-28T03:42:55,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-28T03:42:55,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2c8ad458d2ec9e8bad5d996a8a6cef0, ASSIGN in 325 msec 2024-11-28T03:42:55,674 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:42:55,674 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765375674"}]},"ts":"1732765375674"} 2024-11-28T03:42:55,675 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T03:42:55,676 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:42:55,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-28T03:42:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-28T03:42:56,677 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:56,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b44e9d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,679 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2939af8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,681 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41a700f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,682 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T03:42:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 3a546af21488,38327,1732765248628 2024-11-28T03:42:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7752c611 2024-11-28T03:42:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T03:42:56,686 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T03:42:56,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e758a2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,687 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f85be25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,689 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694d71f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ec0442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,690 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2318ebfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,692 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bc678ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e23aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,693 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6845836e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,694 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57670dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161ef32b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,696 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec63c3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,697 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a708c5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2adec1d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,699 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3281375a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,700 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60119ce1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac88004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,702 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d551856, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,703 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4604fbb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3745b7d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,704 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b06451d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,706 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13267d47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5408ab35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,707 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65533f9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,709 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff8a434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14fc6a2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,710 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195e985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,712 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62e1e52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71575b6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:42:56,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:42:56,714 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:42:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:42:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:42:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f13371, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:42:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:42:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:42:56,716 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:42:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fd2fb7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:42:56,717 DEBUG [Thread-2203 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,717 DEBUG [Thread-2205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,717 DEBUG [Thread-2207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,717 DEBUG [Thread-2218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,717 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:56,717 DEBUG [Thread-2210 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-28T03:42:56,719 DEBUG [Thread-2199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T03:42:56,721 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:56,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:56,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,721 DEBUG [Thread-2201 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,722 DEBUG [Thread-2212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:42:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,722 DEBUG [Thread-2212 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,723 DEBUG [Thread-2212 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:42:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:42:56,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:42:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:42:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:42:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:42:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:42:56,732 DEBUG [Thread-2214 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:56,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T03:42:56,733 DEBUG [Thread-2214 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,734 DEBUG [Thread-2214 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:42:56,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:42:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:42:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:42:56,752 DEBUG [Thread-2216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:42:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:42:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:42:56,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:42:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2] 2024-11-28T03:42:56,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/9d50a5e94259465481448a4fdd34b6df is 50, key is test_row_0/A:col10/1732765376723/Put/seqid=0 2024-11-28T03:42:56,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765436760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:56,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765436762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742320_1496 (size=16681) 2024-11-28T03:42:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:56,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765436763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:56,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/9d50a5e94259465481448a4fdd34b6df 2024-11-28T03:42:56,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765436773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:56,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765436774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:56,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/e7d14e54cfb04ffd8dba9f747f41ea98 is 50, key is test_row_0/B:col10/1732765376723/Put/seqid=0 2024-11-28T03:42:56,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742321_1497 (size=12001) 2024-11-28T03:42:56,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/e7d14e54cfb04ffd8dba9f747f41ea98 2024-11-28T03:42:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T03:42:56,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5143253c79574eb586605aaf2a9a433b is 50, key is test_row_0/C:col10/1732765376723/Put/seqid=0 2024-11-28T03:42:56,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742322_1498 (size=12001) 2024-11-28T03:42:56,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T03:42:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:42:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:56,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:56,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765436876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765436886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765436887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:56,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765436896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:56,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765436896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T03:42:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:42:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:57,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T03:42:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765437087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765437106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765437106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765437106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765437106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T03:42:57,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:57,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:42:57,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:57,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:57,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:57,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5143253c79574eb586605aaf2a9a433b 2024-11-28T03:42:57,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/9d50a5e94259465481448a4fdd34b6df as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d50a5e94259465481448a4fdd34b6df 2024-11-28T03:42:57,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d50a5e94259465481448a4fdd34b6df, entries=250, sequenceid=13, filesize=16.3 K 2024-11-28T03:42:57,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/e7d14e54cfb04ffd8dba9f747f41ea98 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e7d14e54cfb04ffd8dba9f747f41ea98 2024-11-28T03:42:57,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e7d14e54cfb04ffd8dba9f747f41ea98, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T03:42:57,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5143253c79574eb586605aaf2a9a433b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5143253c79574eb586605aaf2a9a433b 2024-11-28T03:42:57,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5143253c79574eb586605aaf2a9a433b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T03:42:57,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 527ms, sequenceid=13, compaction requested=false 2024-11-28T03:42:57,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:57,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T03:42:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:57,337 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T03:42:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:42:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:42:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:42:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:57,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/c45246a53ef7450b9f5d5d1161acfbda is 50, key is test_row_0/A:col10/1732765376773/Put/seqid=0 2024-11-28T03:42:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T03:42:57,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742323_1499 (size=12001) 2024-11-28T03:42:57,355 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/c45246a53ef7450b9f5d5d1161acfbda 2024-11-28T03:42:57,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/e79965e9e84142f1bbef24753fc65c55 is 50, key is test_row_0/B:col10/1732765376773/Put/seqid=0 2024-11-28T03:42:57,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742324_1500 (size=12001) 2024-11-28T03:42:57,398 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/e79965e9e84142f1bbef24753fc65c55 2024-11-28T03:42:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:42:57,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/082357dfcf5f462ea78df07b69e52fa7 is 50, key is test_row_0/C:col10/1732765376773/Put/seqid=0 2024-11-28T03:42:57,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742325_1501 (size=12001) 2024-11-28T03:42:57,435 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/082357dfcf5f462ea78df07b69e52fa7 2024-11-28T03:42:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/c45246a53ef7450b9f5d5d1161acfbda as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c45246a53ef7450b9f5d5d1161acfbda 2024-11-28T03:42:57,448 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c45246a53ef7450b9f5d5d1161acfbda, entries=150, sequenceid=33, filesize=11.7 K 2024-11-28T03:42:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/e79965e9e84142f1bbef24753fc65c55 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e79965e9e84142f1bbef24753fc65c55 2024-11-28T03:42:57,452 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e79965e9e84142f1bbef24753fc65c55, entries=150, sequenceid=33, filesize=11.7 K 2024-11-28T03:42:57,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/082357dfcf5f462ea78df07b69e52fa7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/082357dfcf5f462ea78df07b69e52fa7 2024-11-28T03:42:57,457 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/082357dfcf5f462ea78df07b69e52fa7, entries=150, sequenceid=33, filesize=11.7 K 2024-11-28T03:42:57,458 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=53.67 KB/54960 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 121ms, sequenceid=33, compaction requested=false 2024-11-28T03:42:57,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:57,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:57,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-28T03:42:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-28T03:42:57,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T03:42:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:57,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-28T03:42:57,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 741 msec 2024-11-28T03:42:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:42:57,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:57,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:42:57,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:57,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:42:57,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:57,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 747 msec 2024-11-28T03:42:57,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/3b0b28f02843434e875a01e569d4b650 is 50, key is test_row_0/A:col10/1732765377462/Put/seqid=0 2024-11-28T03:42:57,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742326_1502 (size=14341) 2024-11-28T03:42:57,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/3b0b28f02843434e875a01e569d4b650 2024-11-28T03:42:57,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/d92c53ca08ff4062a70794b6bac84506 is 50, key is test_row_0/B:col10/1732765377462/Put/seqid=0 2024-11-28T03:42:57,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742327_1503 (size=12001) 2024-11-28T03:42:57,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/d92c53ca08ff4062a70794b6bac84506 2024-11-28T03:42:57,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/6a514c9e907d4373b9c49de5a1ff36de is 50, key is test_row_0/C:col10/1732765377462/Put/seqid=0 2024-11-28T03:42:57,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765437504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765437506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742328_1504 (size=12001) 2024-11-28T03:42:57,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765437517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765437521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765437522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765437627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765437627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765437636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765437646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765437646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765437846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765437846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T03:42:57,857 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:57,857 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-28T03:42:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T03:42:57,859 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:57,860 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:57,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:57,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765437856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765437856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765437857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:57,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/6a514c9e907d4373b9c49de5a1ff36de 2024-11-28T03:42:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/3b0b28f02843434e875a01e569d4b650 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3b0b28f02843434e875a01e569d4b650 2024-11-28T03:42:57,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3b0b28f02843434e875a01e569d4b650, entries=200, sequenceid=47, filesize=14.0 K 2024-11-28T03:42:57,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/d92c53ca08ff4062a70794b6bac84506 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d92c53ca08ff4062a70794b6bac84506 2024-11-28T03:42:57,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d92c53ca08ff4062a70794b6bac84506, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T03:42:57,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/6a514c9e907d4373b9c49de5a1ff36de as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6a514c9e907d4373b9c49de5a1ff36de 2024-11-28T03:42:57,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6a514c9e907d4373b9c49de5a1ff36de, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T03:42:57,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 479ms, sequenceid=47, compaction requested=true 2024-11-28T03:42:57,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:57,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:57,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:57,942 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:57,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:57,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:57,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:57,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:57,942 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:57,943 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:57,943 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:57,943 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:42:57,943 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:42:57,943 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:57,943 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:57,943 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d50a5e94259465481448a4fdd34b6df, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c45246a53ef7450b9f5d5d1161acfbda, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3b0b28f02843434e875a01e569d4b650] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=42.0 K 2024-11-28T03:42:57,943 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e7d14e54cfb04ffd8dba9f747f41ea98, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e79965e9e84142f1bbef24753fc65c55, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d92c53ca08ff4062a70794b6bac84506] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=35.2 K 2024-11-28T03:42:57,946 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d50a5e94259465481448a4fdd34b6df, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732765376723 2024-11-28T03:42:57,946 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e7d14e54cfb04ffd8dba9f747f41ea98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732765376723 2024-11-28T03:42:57,946 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e79965e9e84142f1bbef24753fc65c55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732765376751 2024-11-28T03:42:57,946 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45246a53ef7450b9f5d5d1161acfbda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732765376751 2024-11-28T03:42:57,946 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b0b28f02843434e875a01e569d4b650, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732765377439 2024-11-28T03:42:57,946 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d92c53ca08ff4062a70794b6bac84506, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732765377459 2024-11-28T03:42:57,954 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:57,954 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:57,954 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/f7bb804e42f3493d87b07ea05d34cc52 is 50, key is test_row_0/B:col10/1732765377462/Put/seqid=0 2024-11-28T03:42:57,954 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/2b0b3752e4df4b04ba0ce94abbc85165 is 50, key is test_row_0/A:col10/1732765377462/Put/seqid=0 2024-11-28T03:42:57,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742330_1506 (size=12104) 2024-11-28T03:42:57,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T03:42:57,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742329_1505 (size=12104) 2024-11-28T03:42:57,971 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/f7bb804e42f3493d87b07ea05d34cc52 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/f7bb804e42f3493d87b07ea05d34cc52 2024-11-28T03:42:57,975 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into f7bb804e42f3493d87b07ea05d34cc52(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:57,975 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:57,975 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765377942; duration=0sec 2024-11-28T03:42:57,976 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:57,976 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:42:57,976 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:57,980 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:57,980 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:42:57,980 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:57,980 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5143253c79574eb586605aaf2a9a433b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/082357dfcf5f462ea78df07b69e52fa7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6a514c9e907d4373b9c49de5a1ff36de] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=35.2 K 2024-11-28T03:42:57,980 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5143253c79574eb586605aaf2a9a433b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732765376723 2024-11-28T03:42:57,980 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 082357dfcf5f462ea78df07b69e52fa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732765376751 2024-11-28T03:42:57,981 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a514c9e907d4373b9c49de5a1ff36de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732765377459 2024-11-28T03:42:57,989 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:57,990 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/7423efab26cb413bab1d8d086b75aff9 is 50, key is test_row_0/C:col10/1732765377462/Put/seqid=0 2024-11-28T03:42:57,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742331_1507 (size=12104) 2024-11-28T03:42:58,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T03:42:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:58,012 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:42:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:42:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:42:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:42:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/62ea1ca62e684ff6ae4ee323d5cce224 is 50, key is test_row_0/A:col10/1732765377514/Put/seqid=0 2024-11-28T03:42:58,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742332_1508 (size=12001) 2024-11-28T03:42:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:58,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:42:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T03:42:58,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765438182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765438187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765438194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765438194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765438195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765438306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765438316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765438317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765438317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765438317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,364 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/2b0b3752e4df4b04ba0ce94abbc85165 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2b0b3752e4df4b04ba0ce94abbc85165 2024-11-28T03:42:58,371 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 2b0b3752e4df4b04ba0ce94abbc85165(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:58,371 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:58,371 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765377942; duration=0sec 2024-11-28T03:42:58,371 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:58,371 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:42:58,399 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/7423efab26cb413bab1d8d086b75aff9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/7423efab26cb413bab1d8d086b75aff9 2024-11-28T03:42:58,403 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 7423efab26cb413bab1d8d086b75aff9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:58,403 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:58,403 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765377942; duration=0sec 2024-11-28T03:42:58,403 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:58,403 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:42:58,429 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/62ea1ca62e684ff6ae4ee323d5cce224 2024-11-28T03:42:58,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/0f130531ca39472faec932d42211d4ad is 50, key is test_row_0/B:col10/1732765377514/Put/seqid=0 2024-11-28T03:42:58,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742333_1509 (size=12001) 2024-11-28T03:42:58,456 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/0f130531ca39472faec932d42211d4ad 2024-11-28T03:42:58,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/e59dbf02e97e448c9383b2728f682b4c is 50, key is test_row_0/C:col10/1732765377514/Put/seqid=0 2024-11-28T03:42:58,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742334_1510 (size=12001) 2024-11-28T03:42:58,471 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/e59dbf02e97e448c9383b2728f682b4c 2024-11-28T03:42:58,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/62ea1ca62e684ff6ae4ee323d5cce224 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/62ea1ca62e684ff6ae4ee323d5cce224 2024-11-28T03:42:58,478 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/62ea1ca62e684ff6ae4ee323d5cce224, entries=150, sequenceid=67, filesize=11.7 K 2024-11-28T03:42:58,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/0f130531ca39472faec932d42211d4ad as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0f130531ca39472faec932d42211d4ad 2024-11-28T03:42:58,483 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0f130531ca39472faec932d42211d4ad, entries=150, sequenceid=67, filesize=11.7 K 2024-11-28T03:42:58,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/e59dbf02e97e448c9383b2728f682b4c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e59dbf02e97e448c9383b2728f682b4c 2024-11-28T03:42:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T03:42:58,486 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e59dbf02e97e448c9383b2728f682b4c, entries=150, sequenceid=67, filesize=11.7 K 2024-11-28T03:42:58,487 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 475ms, sequenceid=67, compaction requested=false 2024-11-28T03:42:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-28T03:42:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-28T03:42:58,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-28T03:42:58,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 631 msec 2024-11-28T03:42:58,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 636 msec 2024-11-28T03:42:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:42:58,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:42:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:42:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:42:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:42:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:58,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/190752297ba145658b409c8ffcab6772 is 50, key is test_row_0/A:col10/1732765378521/Put/seqid=0 2024-11-28T03:42:58,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742335_1511 (size=14341) 2024-11-28T03:42:58,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765438593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765438595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765438596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765438610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765438611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765438716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765438716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765438716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765438716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765438727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/190752297ba145658b409c8ffcab6772 2024-11-28T03:42:58,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765438926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765438926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765438926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765438927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/66741cc480e742be99fa9ac29fbde53f is 50, key is test_row_0/B:col10/1732765378521/Put/seqid=0 2024-11-28T03:42:58,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742336_1512 (size=12001) 2024-11-28T03:42:58,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765438946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:58,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T03:42:58,997 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:42:58,997 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:42:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-28T03:42:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T03:42:58,999 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:42:59,000 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:42:59,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:42:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T03:42:59,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T03:42:59,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:42:59,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765439246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765439247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:59,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765439247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:59,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765439247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:59,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765439256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:59,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T03:42:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:42:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T03:42:59,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/66741cc480e742be99fa9ac29fbde53f 2024-11-28T03:42:59,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/e63a2554cfbc48919b6d10e22586a0f4 is 50, key is test_row_0/C:col10/1732765378521/Put/seqid=0 2024-11-28T03:42:59,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742337_1513 (size=12001) 2024-11-28T03:42:59,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T03:42:59,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:42:59,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,589 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:42:59,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T03:42:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:42:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T03:42:59,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/e63a2554cfbc48919b6d10e22586a0f4 2024-11-28T03:42:59,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T03:42:59,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:42:59,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:42:59,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/190752297ba145658b409c8ffcab6772 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/190752297ba145658b409c8ffcab6772 2024-11-28T03:42:59,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/190752297ba145658b409c8ffcab6772, entries=200, sequenceid=83, filesize=14.0 K 2024-11-28T03:42:59,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/66741cc480e742be99fa9ac29fbde53f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/66741cc480e742be99fa9ac29fbde53f 2024-11-28T03:42:59,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/66741cc480e742be99fa9ac29fbde53f, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T03:42:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/e63a2554cfbc48919b6d10e22586a0f4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e63a2554cfbc48919b6d10e22586a0f4 2024-11-28T03:42:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765439767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765439767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:59,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765439767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:59,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765439767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:59,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:42:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765439767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:42:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:42:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:42:59,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e63a2554cfbc48919b6d10e22586a0f4, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T03:42:59,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 1259ms, sequenceid=83, compaction requested=true 2024-11-28T03:42:59,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:59,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:42:59,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:59,781 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:59,781 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:59,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:42:59,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:59,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:42:59,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:59,782 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:59,782 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:59,782 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:42:59,782 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:42:59,782 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,782 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,782 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2b0b3752e4df4b04ba0ce94abbc85165, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/62ea1ca62e684ff6ae4ee323d5cce224, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/190752297ba145658b409c8ffcab6772] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=37.5 K 2024-11-28T03:42:59,782 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/f7bb804e42f3493d87b07ea05d34cc52, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0f130531ca39472faec932d42211d4ad, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/66741cc480e742be99fa9ac29fbde53f] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=35.3 K 2024-11-28T03:42:59,782 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b0b3752e4df4b04ba0ce94abbc85165, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732765377459 2024-11-28T03:42:59,782 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting f7bb804e42f3493d87b07ea05d34cc52, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732765377459 2024-11-28T03:42:59,783 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62ea1ca62e684ff6ae4ee323d5cce224, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732765377514 2024-11-28T03:42:59,783 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f130531ca39472faec932d42211d4ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732765377514 2024-11-28T03:42:59,783 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 190752297ba145658b409c8ffcab6772, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732765378187 2024-11-28T03:42:59,783 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 66741cc480e742be99fa9ac29fbde53f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732765378187 2024-11-28T03:42:59,790 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:59,791 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/69337a3939bf49e2a31dbad74f8762dd is 50, key is test_row_0/A:col10/1732765378521/Put/seqid=0 2024-11-28T03:42:59,794 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:59,794 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/0a55b5d874e5486a8c38c5fc6c02010d is 50, key is test_row_0/B:col10/1732765378521/Put/seqid=0 2024-11-28T03:42:59,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742338_1514 (size=12207) 2024-11-28T03:42:59,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742339_1515 (size=12207) 2024-11-28T03:42:59,808 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/69337a3939bf49e2a31dbad74f8762dd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/69337a3939bf49e2a31dbad74f8762dd 2024-11-28T03:42:59,812 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/0a55b5d874e5486a8c38c5fc6c02010d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0a55b5d874e5486a8c38c5fc6c02010d 2024-11-28T03:42:59,815 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 69337a3939bf49e2a31dbad74f8762dd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:59,815 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:59,815 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765379781; duration=0sec 2024-11-28T03:42:59,815 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:42:59,815 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:42:59,816 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:42:59,818 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:42:59,818 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:42:59,818 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,818 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/7423efab26cb413bab1d8d086b75aff9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e59dbf02e97e448c9383b2728f682b4c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e63a2554cfbc48919b6d10e22586a0f4] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=35.3 K 2024-11-28T03:42:59,819 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7423efab26cb413bab1d8d086b75aff9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732765377459 2024-11-28T03:42:59,819 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 0a55b5d874e5486a8c38c5fc6c02010d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:59,819 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:59,819 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765379781; duration=0sec 2024-11-28T03:42:59,819 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:59,819 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:42:59,819 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e59dbf02e97e448c9383b2728f682b4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732765377514 2024-11-28T03:42:59,819 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e63a2554cfbc48919b6d10e22586a0f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732765378187 2024-11-28T03:42:59,827 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:42:59,828 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/487508693b164942a49cc5fb62201524 is 50, key is test_row_0/C:col10/1732765378521/Put/seqid=0 2024-11-28T03:42:59,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742340_1516 (size=12207) 2024-11-28T03:42:59,852 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/487508693b164942a49cc5fb62201524 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/487508693b164942a49cc5fb62201524 2024-11-28T03:42:59,858 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 487508693b164942a49cc5fb62201524(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:42:59,858 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:42:59,858 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765379781; duration=0sec 2024-11-28T03:42:59,858 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:42:59,858 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:42:59,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T03:42:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:42:59,920 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:42:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:42:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:42:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:42:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:42:59,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/984e878f22a041449c8a9a00d55d561d is 50, key is test_row_0/A:col10/1732765378589/Put/seqid=0 2024-11-28T03:42:59,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742341_1517 (size=12001) 2024-11-28T03:43:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T03:43:00,333 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/984e878f22a041449c8a9a00d55d561d 2024-11-28T03:43:00,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/9453e353dc2043738e4130f90f9388bf is 50, key is test_row_0/B:col10/1732765378589/Put/seqid=0 2024-11-28T03:43:00,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742342_1518 (size=12001) 2024-11-28T03:43:00,762 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/9453e353dc2043738e4130f90f9388bf 2024-11-28T03:43:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/e2b21aade3fc4411ba31c779a2797c68 is 50, key is test_row_0/C:col10/1732765378589/Put/seqid=0 2024-11-28T03:43:00,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742343_1519 (size=12001) 2024-11-28T03:43:00,774 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/e2b21aade3fc4411ba31c779a2797c68 2024-11-28T03:43:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/984e878f22a041449c8a9a00d55d561d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/984e878f22a041449c8a9a00d55d561d 2024-11-28T03:43:00,783 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/984e878f22a041449c8a9a00d55d561d, entries=150, sequenceid=104, filesize=11.7 K 2024-11-28T03:43:00,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/9453e353dc2043738e4130f90f9388bf as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/9453e353dc2043738e4130f90f9388bf 2024-11-28T03:43:00,789 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/9453e353dc2043738e4130f90f9388bf, entries=150, sequenceid=104, filesize=11.7 K 2024-11-28T03:43:00,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/e2b21aade3fc4411ba31c779a2797c68 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e2b21aade3fc4411ba31c779a2797c68 2024-11-28T03:43:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:00,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:00,797 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e2b21aade3fc4411ba31c779a2797c68, entries=150, sequenceid=104, filesize=11.7 K 2024-11-28T03:43:00,798 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=46.96 KB/48090 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 878ms, sequenceid=104, compaction requested=false 2024-11-28T03:43:00,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:00,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:00,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-28T03:43:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-28T03:43:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:00,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T03:43:00,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:00,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:00,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:00,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:00,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:00,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:00,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-28T03:43:00,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7990 sec 2024-11-28T03:43:00,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.8060 sec 2024-11-28T03:43:00,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/7f1b8ff923d44ee2ae5c1e97f6de0783 is 50, key is test_row_1/A:col10/1732765380793/Put/seqid=0 2024-11-28T03:43:00,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742344_1520 (size=14337) 2024-11-28T03:43:00,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/7f1b8ff923d44ee2ae5c1e97f6de0783 2024-11-28T03:43:00,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/7c13ff1eb46345c281398d94ca86c685 is 50, key is test_row_1/B:col10/1732765380793/Put/seqid=0 2024-11-28T03:43:00,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742345_1521 (size=9657) 2024-11-28T03:43:00,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765440836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:00,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765380840, max timestamp=1732765380847Num uniques 859; 2024-11-28T03:43:00,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:00,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765380840, max timestamp=1732765380847Num uniques 859; 2024-11-28T03:43:00,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:00,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765380840, max timestamp=1732765380847Num uniques 859; 2024-11-28T03:43:00,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:00,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765440898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:00,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765440901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765440909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:00,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765440913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:00,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765440957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765441016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765441026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765441027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765441027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T03:43:01,156 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:01,157 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-28T03:43:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T03:43:01,158 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:01,159 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:01,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:01,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765441166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765441237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765441237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765441237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765441238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/7c13ff1eb46345c281398d94ca86c685 2024-11-28T03:43:01,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5fab8497b427420c9d5ddd2bd4eebcb6 is 50, key is test_row_1/C:col10/1732765380793/Put/seqid=0 2024-11-28T03:43:01,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742346_1522 (size=9657) 2024-11-28T03:43:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T03:43:01,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T03:43:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:01,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:01,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T03:43:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:01,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:01,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T03:43:01,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765441476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765441546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765441546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765441546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765441546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:01,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T03:43:01,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:01,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:01,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5fab8497b427420c9d5ddd2bd4eebcb6 2024-11-28T03:43:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/7f1b8ff923d44ee2ae5c1e97f6de0783 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7f1b8ff923d44ee2ae5c1e97f6de0783 2024-11-28T03:43:01,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7f1b8ff923d44ee2ae5c1e97f6de0783, entries=200, sequenceid=115, filesize=14.0 K 2024-11-28T03:43:01,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/7c13ff1eb46345c281398d94ca86c685 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/7c13ff1eb46345c281398d94ca86c685 2024-11-28T03:43:01,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/7c13ff1eb46345c281398d94ca86c685, entries=100, sequenceid=115, filesize=9.4 K 2024-11-28T03:43:01,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5fab8497b427420c9d5ddd2bd4eebcb6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5fab8497b427420c9d5ddd2bd4eebcb6 2024-11-28T03:43:01,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5fab8497b427420c9d5ddd2bd4eebcb6, entries=100, sequenceid=115, filesize=9.4 K 2024-11-28T03:43:01,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 876ms, sequenceid=115, compaction requested=true 2024-11-28T03:43:01,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:01,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:01,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:01,676 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:01,676 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:01,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:01,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:01,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:01,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:01,678 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38545 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:01,678 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:01,678 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:43:01,678 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:43:01,678 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:01,678 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:01,678 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0a55b5d874e5486a8c38c5fc6c02010d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/9453e353dc2043738e4130f90f9388bf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/7c13ff1eb46345c281398d94ca86c685] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=33.1 K 2024-11-28T03:43:01,678 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/69337a3939bf49e2a31dbad74f8762dd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/984e878f22a041449c8a9a00d55d561d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7f1b8ff923d44ee2ae5c1e97f6de0783] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=37.6 K 2024-11-28T03:43:01,678 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a55b5d874e5486a8c38c5fc6c02010d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732765378187 2024-11-28T03:43:01,678 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69337a3939bf49e2a31dbad74f8762dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732765378187 2024-11-28T03:43:01,679 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9453e353dc2043738e4130f90f9388bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732765378589 2024-11-28T03:43:01,679 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 984e878f22a041449c8a9a00d55d561d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732765378589 2024-11-28T03:43:01,679 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c13ff1eb46345c281398d94ca86c685, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732765380793 2024-11-28T03:43:01,679 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f1b8ff923d44ee2ae5c1e97f6de0783, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732765380793 2024-11-28T03:43:01,687 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:01,687 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:01,687 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/2d0210f662df4b90ab3ef55655e45ee9 is 50, key is test_row_0/B:col10/1732765378589/Put/seqid=0 2024-11-28T03:43:01,687 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/c07e7bd602b64d59a878ebe063d39f50 is 50, key is test_row_0/A:col10/1732765378589/Put/seqid=0 2024-11-28T03:43:01,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742348_1524 (size=12309) 2024-11-28T03:43:01,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742347_1523 (size=12309) 2024-11-28T03:43:01,698 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/c07e7bd602b64d59a878ebe063d39f50 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c07e7bd602b64d59a878ebe063d39f50 2024-11-28T03:43:01,698 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/2d0210f662df4b90ab3ef55655e45ee9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/2d0210f662df4b90ab3ef55655e45ee9 2024-11-28T03:43:01,709 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into c07e7bd602b64d59a878ebe063d39f50(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:01,709 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:01,709 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765381676; duration=0sec 2024-11-28T03:43:01,709 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:01,709 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:43:01,709 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:01,710 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 2d0210f662df4b90ab3ef55655e45ee9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:01,710 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:01,710 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765381676; duration=0sec 2024-11-28T03:43:01,710 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:01,710 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:43:01,710 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:01,710 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:43:01,710 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:01,710 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/487508693b164942a49cc5fb62201524, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e2b21aade3fc4411ba31c779a2797c68, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5fab8497b427420c9d5ddd2bd4eebcb6] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=33.1 K 2024-11-28T03:43:01,710 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 487508693b164942a49cc5fb62201524, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732765378187 2024-11-28T03:43:01,711 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2b21aade3fc4411ba31c779a2797c68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732765378589 2024-11-28T03:43:01,711 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fab8497b427420c9d5ddd2bd4eebcb6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732765380793 2024-11-28T03:43:01,719 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:01,720 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/a80fbbb17ca04dbb8f07276f4a2ce8cf is 50, key is test_row_0/C:col10/1732765378589/Put/seqid=0 2024-11-28T03:43:01,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742349_1525 (size=12309) 2024-11-28T03:43:01,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T03:43:01,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:01,777 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-28T03:43:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T03:43:01,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/d41201541fe944b2a56d23ec850c6e62 is 50, key is test_row_0/A:col10/1732765380911/Put/seqid=0 2024-11-28T03:43:01,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742350_1526 (size=12151) 2024-11-28T03:43:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:02,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:02,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765442071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765442081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765442082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765442083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765442084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,133 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/a80fbbb17ca04dbb8f07276f4a2ce8cf as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/a80fbbb17ca04dbb8f07276f4a2ce8cf 2024-11-28T03:43:02,137 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into a80fbbb17ca04dbb8f07276f4a2ce8cf(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:02,137 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:02,137 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765381676; duration=0sec 2024-11-28T03:43:02,138 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:02,138 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:43:02,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765442186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,217 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/d41201541fe944b2a56d23ec850c6e62 2024-11-28T03:43:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765442206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765442206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765442206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765442206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/26862949bff7458785b334bef89b447b is 50, key is test_row_0/B:col10/1732765380911/Put/seqid=0 2024-11-28T03:43:02,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742351_1527 (size=12151) 2024-11-28T03:43:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T03:43:02,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765442396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765442426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765442426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765442426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765442426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,629 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/26862949bff7458785b334bef89b447b 2024-11-28T03:43:02,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5c30d6b02d914ffb8d82774240c9d93a is 50, key is test_row_0/C:col10/1732765380911/Put/seqid=0 2024-11-28T03:43:02,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742352_1528 (size=12151) 2024-11-28T03:43:02,641 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5c30d6b02d914ffb8d82774240c9d93a 2024-11-28T03:43:02,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/d41201541fe944b2a56d23ec850c6e62 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d41201541fe944b2a56d23ec850c6e62 2024-11-28T03:43:02,649 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d41201541fe944b2a56d23ec850c6e62, entries=150, sequenceid=153, filesize=11.9 K 2024-11-28T03:43:02,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/26862949bff7458785b334bef89b447b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/26862949bff7458785b334bef89b447b 2024-11-28T03:43:02,653 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/26862949bff7458785b334bef89b447b, entries=150, sequenceid=153, filesize=11.9 K 2024-11-28T03:43:02,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5c30d6b02d914ffb8d82774240c9d93a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5c30d6b02d914ffb8d82774240c9d93a 2024-11-28T03:43:02,657 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5c30d6b02d914ffb8d82774240c9d93a, entries=150, sequenceid=153, filesize=11.9 K 2024-11-28T03:43:02,658 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 882ms, sequenceid=153, compaction requested=false 2024-11-28T03:43:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-28T03:43:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-28T03:43:02,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-28T03:43:02,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-11-28T03:43:02,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.5060 sec 2024-11-28T03:43:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:02,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T03:43:02,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:02,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/257465698dcd4f36b6eb0650953cad10 is 50, key is test_row_0/A:col10/1732765382711/Put/seqid=0 2024-11-28T03:43:02,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742353_1529 (size=14541) 2024-11-28T03:43:02,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/257465698dcd4f36b6eb0650953cad10 2024-11-28T03:43:02,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/645ded5a2cc04a04854df20099600a38 is 50, key is test_row_0/B:col10/1732765382711/Put/seqid=0 2024-11-28T03:43:02,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742354_1530 (size=12151) 2024-11-28T03:43:02,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765442763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765442765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765442766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765442777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765442780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765442886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765442886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765442887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765442896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:02,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765442906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765443106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765443107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765443107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765443107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765443126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/645ded5a2cc04a04854df20099600a38 2024-11-28T03:43:03,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8c8e811f1cd24f5fb18dced88e8103ef is 50, key is test_row_0/C:col10/1732765382711/Put/seqid=0 2024-11-28T03:43:03,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742355_1531 (size=12151) 2024-11-28T03:43:03,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8c8e811f1cd24f5fb18dced88e8103ef 2024-11-28T03:43:03,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/257465698dcd4f36b6eb0650953cad10 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/257465698dcd4f36b6eb0650953cad10 2024-11-28T03:43:03,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/257465698dcd4f36b6eb0650953cad10, entries=200, sequenceid=167, filesize=14.2 K 2024-11-28T03:43:03,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/645ded5a2cc04a04854df20099600a38 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/645ded5a2cc04a04854df20099600a38 2024-11-28T03:43:03,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/645ded5a2cc04a04854df20099600a38, entries=150, sequenceid=167, filesize=11.9 K 2024-11-28T03:43:03,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8c8e811f1cd24f5fb18dced88e8103ef as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c8e811f1cd24f5fb18dced88e8103ef 2024-11-28T03:43:03,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c8e811f1cd24f5fb18dced88e8103ef, entries=150, sequenceid=167, filesize=11.9 K 2024-11-28T03:43:03,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 483ms, sequenceid=167, compaction requested=true 2024-11-28T03:43:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:03,196 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:03,197 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:03,197 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:03,197 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:43:03,198 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:03,198 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c07e7bd602b64d59a878ebe063d39f50, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d41201541fe944b2a56d23ec850c6e62, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/257465698dcd4f36b6eb0650953cad10] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=38.1 K 2024-11-28T03:43:03,198 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:03,198 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:43:03,198 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:03,198 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/2d0210f662df4b90ab3ef55655e45ee9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/26862949bff7458785b334bef89b447b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/645ded5a2cc04a04854df20099600a38] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=35.8 K 2024-11-28T03:43:03,198 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c07e7bd602b64d59a878ebe063d39f50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732765378589 2024-11-28T03:43:03,199 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d0210f662df4b90ab3ef55655e45ee9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732765378589 2024-11-28T03:43:03,199 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d41201541fe944b2a56d23ec850c6e62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732765380880 2024-11-28T03:43:03,199 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 257465698dcd4f36b6eb0650953cad10, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732765382069 2024-11-28T03:43:03,199 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 26862949bff7458785b334bef89b447b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732765380880 2024-11-28T03:43:03,200 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 645ded5a2cc04a04854df20099600a38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732765382069 2024-11-28T03:43:03,212 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:03,213 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/2de80b4e4a694761bfd9826c76a8fc89 is 50, key is test_row_0/A:col10/1732765382711/Put/seqid=0 2024-11-28T03:43:03,217 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#454 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:03,217 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/93cca20a874344158c74c4601910b23a is 50, key is test_row_0/B:col10/1732765382711/Put/seqid=0 2024-11-28T03:43:03,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742357_1533 (size=12561) 2024-11-28T03:43:03,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742356_1532 (size=12561) 2024-11-28T03:43:03,244 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/2de80b4e4a694761bfd9826c76a8fc89 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2de80b4e4a694761bfd9826c76a8fc89 2024-11-28T03:43:03,252 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 2de80b4e4a694761bfd9826c76a8fc89(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:03,253 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:03,253 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765383196; duration=0sec 2024-11-28T03:43:03,253 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:03,253 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:43:03,253 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:03,258 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:03,258 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:43:03,258 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:03,258 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/a80fbbb17ca04dbb8f07276f4a2ce8cf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5c30d6b02d914ffb8d82774240c9d93a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c8e811f1cd24f5fb18dced88e8103ef] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=35.8 K 2024-11-28T03:43:03,258 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a80fbbb17ca04dbb8f07276f4a2ce8cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732765378589 2024-11-28T03:43:03,259 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c30d6b02d914ffb8d82774240c9d93a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732765380880 2024-11-28T03:43:03,259 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c8e811f1cd24f5fb18dced88e8103ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732765382069 2024-11-28T03:43:03,266 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:03,266 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/aeaf8709dba4448899cd3eb244d42a30 is 50, key is test_row_0/C:col10/1732765382711/Put/seqid=0 2024-11-28T03:43:03,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742358_1534 (size=12561) 2024-11-28T03:43:03,278 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/aeaf8709dba4448899cd3eb244d42a30 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/aeaf8709dba4448899cd3eb244d42a30 2024-11-28T03:43:03,284 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into aeaf8709dba4448899cd3eb244d42a30(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:03,284 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:03,284 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765383196; duration=0sec 2024-11-28T03:43:03,284 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:03,284 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:43:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T03:43:03,306 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:03,307 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-28T03:43:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T03:43:03,309 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:03,310 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:03,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T03:43:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:03,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T03:43:03,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:03,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:03,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:03,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:03,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:03,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:03,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/2ad98151756c4019bb30c42087a86647 is 50, key is test_row_0/A:col10/1732765382777/Put/seqid=0 2024-11-28T03:43:03,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742359_1535 (size=14541) 2024-11-28T03:43:03,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/2ad98151756c4019bb30c42087a86647 2024-11-28T03:43:03,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/cee0f1921c5241bd9b434ca87f928569 is 50, key is test_row_0/B:col10/1732765382777/Put/seqid=0 2024-11-28T03:43:03,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742360_1536 (size=12151) 2024-11-28T03:43:03,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/cee0f1921c5241bd9b434ca87f928569 2024-11-28T03:43:03,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765443446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765443446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765443447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T03:43:03,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:03,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:03,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:03,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5ca1089a6ecf4564b095bdafc7cbf3c2 is 50, key is test_row_0/C:col10/1732765382777/Put/seqid=0 2024-11-28T03:43:03,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:03,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:03,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742361_1537 (size=12151) 2024-11-28T03:43:03,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765443455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765443457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765443566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765443566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765443576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765443576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T03:43:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:03,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T03:43:03,642 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/93cca20a874344158c74c4601910b23a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/93cca20a874344158c74c4601910b23a 2024-11-28T03:43:03,648 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 93cca20a874344158c74c4601910b23a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:03,648 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:03,648 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765383196; duration=0sec 2024-11-28T03:43:03,648 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:03,648 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:43:03,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T03:43:03,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:03,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:03,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:03,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:03,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:03,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765443776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765443786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765443797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765443797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:03,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5ca1089a6ecf4564b095bdafc7cbf3c2 2024-11-28T03:43:03,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/2ad98151756c4019bb30c42087a86647 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2ad98151756c4019bb30c42087a86647 2024-11-28T03:43:03,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2ad98151756c4019bb30c42087a86647, entries=200, sequenceid=189, filesize=14.2 K 2024-11-28T03:43:03,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/cee0f1921c5241bd9b434ca87f928569 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cee0f1921c5241bd9b434ca87f928569 2024-11-28T03:43:03,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cee0f1921c5241bd9b434ca87f928569, entries=150, sequenceid=189, filesize=11.9 K 2024-11-28T03:43:03,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5ca1089a6ecf4564b095bdafc7cbf3c2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5ca1089a6ecf4564b095bdafc7cbf3c2 2024-11-28T03:43:03,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5ca1089a6ecf4564b095bdafc7cbf3c2, entries=150, sequenceid=189, filesize=11.9 K 2024-11-28T03:43:03,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 455ms, sequenceid=189, compaction requested=false 2024-11-28T03:43:03,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:03,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T03:43:03,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:03,922 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T03:43:03,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:03,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:03,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:03,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:03,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:03,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:03,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/7407a931bd31492ba203647bb2901565 is 50, key is test_row_1/A:col10/1732765383455/Put/seqid=0 2024-11-28T03:43:03,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742362_1538 (size=9757) 2024-11-28T03:43:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T03:43:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:03,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:04,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765444086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765444093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765444097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765444106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765444116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765444206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,334 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/7407a931bd31492ba203647bb2901565 2024-11-28T03:43:04,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/ed3132dbc1a1470fbe2825aa66fa444e is 50, key is test_row_1/B:col10/1732765383455/Put/seqid=0 2024-11-28T03:43:04,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742363_1539 (size=9757) 2024-11-28T03:43:04,367 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/ed3132dbc1a1470fbe2825aa66fa444e 2024-11-28T03:43:04,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/2b2d639cdc6940d0aef306be4af05eff is 50, key is test_row_1/C:col10/1732765383455/Put/seqid=0 2024-11-28T03:43:04,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742364_1540 (size=9757) 2024-11-28T03:43:04,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765444416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T03:43:04,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765444607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765444616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765444626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765444636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765444736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:04,824 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/2b2d639cdc6940d0aef306be4af05eff 2024-11-28T03:43:04,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/7407a931bd31492ba203647bb2901565 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7407a931bd31492ba203647bb2901565 2024-11-28T03:43:04,831 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7407a931bd31492ba203647bb2901565, entries=100, sequenceid=202, filesize=9.5 K 2024-11-28T03:43:04,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/ed3132dbc1a1470fbe2825aa66fa444e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ed3132dbc1a1470fbe2825aa66fa444e 2024-11-28T03:43:04,835 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ed3132dbc1a1470fbe2825aa66fa444e, entries=100, sequenceid=202, filesize=9.5 K 2024-11-28T03:43:04,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/2b2d639cdc6940d0aef306be4af05eff as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2b2d639cdc6940d0aef306be4af05eff 2024-11-28T03:43:04,839 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2b2d639cdc6940d0aef306be4af05eff, entries=100, sequenceid=202, filesize=9.5 K 2024-11-28T03:43:04,840 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 918ms, sequenceid=202, compaction requested=true 2024-11-28T03:43:04,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:04,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:04,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-28T03:43:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-28T03:43:04,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-28T03:43:04,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5310 sec 2024-11-28T03:43:04,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.5360 sec 2024-11-28T03:43:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:05,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T03:43:05,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:05,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:05,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:05,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:05,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:05,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:05,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/9d40f237156046e8ba942c972858fbb1 is 50, key is test_row_0/A:col10/1732765385258/Put/seqid=0 2024-11-28T03:43:05,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742365_1541 (size=14541) 2024-11-28T03:43:05,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765445331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:05,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765445437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T03:43:05,456 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:05,457 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-28T03:43:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T03:43:05,458 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:05,459 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:05,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T03:43:05,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T03:43:05,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:05,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:05,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:05,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:05,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:05,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765445626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:05,639 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:05,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765445637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:05,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765445637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:05,641 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:05,641 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_0' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:05,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765445656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:05,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765445656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:05,669 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:05,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/9d40f237156046e8ba942c972858fbb1 2024-11-28T03:43:05,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/4f0c494e09044fc4937f2ff42e6b4ca3 is 50, key is test_row_0/B:col10/1732765385258/Put/seqid=0 2024-11-28T03:43:05,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742366_1542 (size=12151) 2024-11-28T03:43:05,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/4f0c494e09044fc4937f2ff42e6b4ca3 2024-11-28T03:43:05,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5bc4bf80f8374e32b0de829452e066c4 is 50, key is test_row_0/C:col10/1732765385258/Put/seqid=0 2024-11-28T03:43:05,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742367_1543 (size=12151) 2024-11-28T03:43:05,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5bc4bf80f8374e32b0de829452e066c4 2024-11-28T03:43:05,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/9d40f237156046e8ba942c972858fbb1 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d40f237156046e8ba942c972858fbb1 2024-11-28T03:43:05,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d40f237156046e8ba942c972858fbb1, entries=200, sequenceid=223, filesize=14.2 K 2024-11-28T03:43:05,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/4f0c494e09044fc4937f2ff42e6b4ca3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4f0c494e09044fc4937f2ff42e6b4ca3 2024-11-28T03:43:05,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4f0c494e09044fc4937f2ff42e6b4ca3, entries=150, sequenceid=223, filesize=11.9 K 2024-11-28T03:43:05,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5bc4bf80f8374e32b0de829452e066c4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5bc4bf80f8374e32b0de829452e066c4 2024-11-28T03:43:05,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5bc4bf80f8374e32b0de829452e066c4, entries=150, sequenceid=223, filesize=11.9 K 2024-11-28T03:43:05,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 455ms, sequenceid=223, compaction requested=true 2024-11-28T03:43:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:05,716 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:43:05,716 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:43:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:05,718 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:43:05,718 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:43:05,719 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:05,719 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/93cca20a874344158c74c4601910b23a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cee0f1921c5241bd9b434ca87f928569, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ed3132dbc1a1470fbe2825aa66fa444e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4f0c494e09044fc4937f2ff42e6b4ca3] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=45.5 K 2024-11-28T03:43:05,719 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:43:05,719 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:43:05,719 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 93cca20a874344158c74c4601910b23a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732765382069 2024-11-28T03:43:05,719 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:05,719 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2de80b4e4a694761bfd9826c76a8fc89, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2ad98151756c4019bb30c42087a86647, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7407a931bd31492ba203647bb2901565, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d40f237156046e8ba942c972858fbb1] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=50.2 K 2024-11-28T03:43:05,719 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting cee0f1921c5241bd9b434ca87f928569, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732765382775 2024-11-28T03:43:05,719 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de80b4e4a694761bfd9826c76a8fc89, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732765382069 2024-11-28T03:43:05,720 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ed3132dbc1a1470fbe2825aa66fa444e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732765383452 2024-11-28T03:43:05,720 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad98151756c4019bb30c42087a86647, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732765382761 2024-11-28T03:43:05,720 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f0c494e09044fc4937f2ff42e6b4ca3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732765384057 2024-11-28T03:43:05,720 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7407a931bd31492ba203647bb2901565, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732765383452 2024-11-28T03:43:05,720 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d40f237156046e8ba942c972858fbb1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732765384057 2024-11-28T03:43:05,727 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:05,728 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/46e1a0fdd95e40b7a402991df34e4dd5 is 50, key is test_row_0/B:col10/1732765385258/Put/seqid=0 2024-11-28T03:43:05,730 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:05,731 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/1f20ca871f254acba9e94e5ea6372ba5 is 50, key is test_row_0/A:col10/1732765385258/Put/seqid=0 2024-11-28T03:43:05,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742368_1544 (size=12697) 2024-11-28T03:43:05,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742369_1545 (size=12697) 2024-11-28T03:43:05,750 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/46e1a0fdd95e40b7a402991df34e4dd5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/46e1a0fdd95e40b7a402991df34e4dd5 2024-11-28T03:43:05,755 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 46e1a0fdd95e40b7a402991df34e4dd5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:05,755 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:05,755 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=12, startTime=1732765385716; duration=0sec 2024-11-28T03:43:05,755 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:05,755 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:43:05,755 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T03:43:05,756 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T03:43:05,756 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:43:05,756 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:05,756 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/aeaf8709dba4448899cd3eb244d42a30, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5ca1089a6ecf4564b095bdafc7cbf3c2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2b2d639cdc6940d0aef306be4af05eff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5bc4bf80f8374e32b0de829452e066c4] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=45.5 K 2024-11-28T03:43:05,758 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting aeaf8709dba4448899cd3eb244d42a30, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732765382069 2024-11-28T03:43:05,759 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca1089a6ecf4564b095bdafc7cbf3c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732765382775 2024-11-28T03:43:05,759 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2d639cdc6940d0aef306be4af05eff, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732765383452 2024-11-28T03:43:05,759 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bc4bf80f8374e32b0de829452e066c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732765384057 2024-11-28T03:43:05,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T03:43:05,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:05,766 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T03:43:05,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:05,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:05,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:05,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:05,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:05,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:05,768 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:05,769 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/d0a983c851d74d7091e57a653d8f9d48 is 50, key is test_row_0/C:col10/1732765385258/Put/seqid=0 2024-11-28T03:43:05,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742370_1546 (size=12697) 2024-11-28T03:43:05,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/3193b8fcecdf4132baa840b99c2b0971 is 50, key is test_row_0/A:col10/1732765385294/Put/seqid=0 2024-11-28T03:43:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T03:43:05,789 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/d0a983c851d74d7091e57a653d8f9d48 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/d0a983c851d74d7091e57a653d8f9d48 2024-11-28T03:43:05,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742371_1547 (size=12151) 2024-11-28T03:43:05,802 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/3193b8fcecdf4132baa840b99c2b0971 2024-11-28T03:43:05,806 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into d0a983c851d74d7091e57a653d8f9d48(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:05,806 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:05,806 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=12, startTime=1732765385716; duration=0sec 2024-11-28T03:43:05,807 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:05,807 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:43:05,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/cf90fbc4bc0945bf8b31ce82d05f2faf is 50, key is test_row_0/B:col10/1732765385294/Put/seqid=0 2024-11-28T03:43:05,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742372_1548 (size=12151) 2024-11-28T03:43:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:05,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T03:43:06,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765386074, max timestamp=1732765386121Num uniques 859; 2024-11-28T03:43:06,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:06,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765386074, max timestamp=1732765386121Num uniques 859; 2024-11-28T03:43:06,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:06,145 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/1f20ca871f254acba9e94e5ea6372ba5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/1f20ca871f254acba9e94e5ea6372ba5 2024-11-28T03:43:06,149 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 1f20ca871f254acba9e94e5ea6372ba5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:06,149 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:06,149 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=12, startTime=1732765385716; duration=0sec 2024-11-28T03:43:06,149 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:06,149 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:43:06,216 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/cf90fbc4bc0945bf8b31ce82d05f2faf 2024-11-28T03:43:06,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/bf4d93c104404f3284f6f0d999072861 is 50, key is test_row_0/C:col10/1732765385294/Put/seqid=0 2024-11-28T03:43:06,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742373_1549 (size=12151) 2024-11-28T03:43:06,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765446222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:06,240 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/bf4d93c104404f3284f6f0d999072861 2024-11-28T03:43:06,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/3193b8fcecdf4132baa840b99c2b0971 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3193b8fcecdf4132baa840b99c2b0971 2024-11-28T03:43:06,248 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3193b8fcecdf4132baa840b99c2b0971, entries=150, sequenceid=235, filesize=11.9 K 2024-11-28T03:43:06,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/cf90fbc4bc0945bf8b31ce82d05f2faf as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cf90fbc4bc0945bf8b31ce82d05f2faf 2024-11-28T03:43:06,256 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cf90fbc4bc0945bf8b31ce82d05f2faf, entries=150, sequenceid=235, filesize=11.9 K 2024-11-28T03:43:06,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/bf4d93c104404f3284f6f0d999072861 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/bf4d93c104404f3284f6f0d999072861 2024-11-28T03:43:06,262 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/bf4d93c104404f3284f6f0d999072861, entries=150, sequenceid=235, filesize=11.9 K 2024-11-28T03:43:06,267 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 502ms, sequenceid=235, compaction requested=false 2024-11-28T03:43:06,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:06,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:06,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-28T03:43:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-28T03:43:06,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-28T03:43:06,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 810 msec 2024-11-28T03:43:06,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 817 msec 2024-11-28T03:43:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:06,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=131.94 KB heapSize=383.31 KB 2024-11-28T03:43:06,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:06,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:06,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:06,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:06,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:06,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:06,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/52e33a414c364985985b72a11037de42 is 50, key is test_row_0/A:col10/1732765386206/Put/seqid=0 2024-11-28T03:43:06,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742374_1550 (size=14741) 2024-11-28T03:43:06,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/52e33a414c364985985b72a11037de42 2024-11-28T03:43:06,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/eb7bd3560ea14988b6c53bceb8d6473c is 50, key is test_row_0/B:col10/1732765386206/Put/seqid=0 2024-11-28T03:43:06,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742375_1551 (size=12301) 2024-11-28T03:43:06,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/eb7bd3560ea14988b6c53bceb8d6473c 2024-11-28T03:43:06,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/4faa3267b3424161b213252cdb39d3c6 is 50, key is test_row_0/C:col10/1732765386206/Put/seqid=0 2024-11-28T03:43:06,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742376_1552 (size=12301) 2024-11-28T03:43:06,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765446448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:06,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765446576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T03:43:06,596 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:06,597 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-28T03:43:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T03:43:06,598 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:06,600 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:06,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T03:43:06,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-28T03:43:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:06,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:06,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/4faa3267b3424161b213252cdb39d3c6 2024-11-28T03:43:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/52e33a414c364985985b72a11037de42 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52e33a414c364985985b72a11037de42 2024-11-28T03:43:06,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52e33a414c364985985b72a11037de42, entries=200, sequenceid=269, filesize=14.4 K 2024-11-28T03:43:06,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/eb7bd3560ea14988b6c53bceb8d6473c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/eb7bd3560ea14988b6c53bceb8d6473c 2024-11-28T03:43:06,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765446786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:06,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/eb7bd3560ea14988b6c53bceb8d6473c, entries=150, sequenceid=269, filesize=12.0 K 2024-11-28T03:43:06,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/4faa3267b3424161b213252cdb39d3c6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4faa3267b3424161b213252cdb39d3c6 2024-11-28T03:43:06,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4faa3267b3424161b213252cdb39d3c6, entries=150, sequenceid=269, filesize=12.0 K 2024-11-28T03:43:06,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~383.27 KB/392464, currentSize=46.96 KB/48090 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 457ms, sequenceid=269, compaction requested=true 2024-11-28T03:43:06,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:06,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:06,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:06,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:06,811 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:06,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:06,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:06,811 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:06,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:06,811 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:06,811 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:06,812 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:43:06,812 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:06,812 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/46e1a0fdd95e40b7a402991df34e4dd5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cf90fbc4bc0945bf8b31ce82d05f2faf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/eb7bd3560ea14988b6c53bceb8d6473c] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=36.3 K 2024-11-28T03:43:06,812 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:43:06,812 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:06,812 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/1f20ca871f254acba9e94e5ea6372ba5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3193b8fcecdf4132baa840b99c2b0971, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52e33a414c364985985b72a11037de42] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=38.7 K 2024-11-28T03:43:06,815 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e1a0fdd95e40b7a402991df34e4dd5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732765384057 2024-11-28T03:43:06,815 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f20ca871f254acba9e94e5ea6372ba5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732765384057 2024-11-28T03:43:06,815 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting cf90fbc4bc0945bf8b31ce82d05f2faf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732765385294 2024-11-28T03:43:06,815 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3193b8fcecdf4132baa840b99c2b0971, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732765385294 2024-11-28T03:43:06,815 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52e33a414c364985985b72a11037de42, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732765386187 2024-11-28T03:43:06,815 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting eb7bd3560ea14988b6c53bceb8d6473c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732765386206 2024-11-28T03:43:06,834 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:06,834 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/32beaabbf4e34226a7348afaf3c74a9d is 50, key is test_row_0/B:col10/1732765386206/Put/seqid=0 2024-11-28T03:43:06,837 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:06,837 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/addf0d8df24a4800ae080dbb53411149 is 50, key is test_row_0/A:col10/1732765386206/Put/seqid=0 2024-11-28T03:43:06,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742377_1553 (size=12949) 2024-11-28T03:43:06,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742378_1554 (size=12949) 2024-11-28T03:43:06,848 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/32beaabbf4e34226a7348afaf3c74a9d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/32beaabbf4e34226a7348afaf3c74a9d 2024-11-28T03:43:06,853 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 32beaabbf4e34226a7348afaf3c74a9d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:06,853 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:06,853 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765386811; duration=0sec 2024-11-28T03:43:06,853 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:06,854 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:43:06,854 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:06,854 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:06,854 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:43:06,854 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:06,855 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/d0a983c851d74d7091e57a653d8f9d48, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/bf4d93c104404f3284f6f0d999072861, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4faa3267b3424161b213252cdb39d3c6] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=36.3 K 2024-11-28T03:43:06,855 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a983c851d74d7091e57a653d8f9d48, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732765384057 2024-11-28T03:43:06,855 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting bf4d93c104404f3284f6f0d999072861, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732765385294 2024-11-28T03:43:06,855 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4faa3267b3424161b213252cdb39d3c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732765386206 2024-11-28T03:43:06,864 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:06,865 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/6e96c8d855bb4351ad28134fdc98b01c is 50, key is test_row_0/C:col10/1732765386206/Put/seqid=0 2024-11-28T03:43:06,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742379_1555 (size=12949) 2024-11-28T03:43:06,878 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/6e96c8d855bb4351ad28134fdc98b01c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6e96c8d855bb4351ad28134fdc98b01c 2024-11-28T03:43:06,883 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 6e96c8d855bb4351ad28134fdc98b01c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:06,883 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:06,883 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765386811; duration=0sec 2024-11-28T03:43:06,883 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:06,883 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:43:06,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-28T03:43:06,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:06,906 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:43:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:06,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/61c26fb5bfec400db1c370ef9c573b0d is 50, key is test_row_1/A:col10/1732765386394/Put/seqid=0 2024-11-28T03:43:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T03:43:06,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742380_1556 (size=9857) 2024-11-28T03:43:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:07,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T03:43:07,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765387141, max timestamp=1732765387226Num uniques 859; 2024-11-28T03:43:07,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765387141, max timestamp=1732765387226Num uniques 859; 2024-11-28T03:43:07,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:07,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:07,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765387141, max timestamp=1732765387226Num uniques 859; 2024-11-28T03:43:07,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:07,248 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/addf0d8df24a4800ae080dbb53411149 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/addf0d8df24a4800ae080dbb53411149 2024-11-28T03:43:07,252 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into addf0d8df24a4800ae080dbb53411149(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:07,252 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:07,253 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765386810; duration=0sec 2024-11-28T03:43:07,253 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:07,253 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:43:07,321 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/61c26fb5bfec400db1c370ef9c573b0d 2024-11-28T03:43:07,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/c75c7c78e7c74b619719afee20aa7977 is 50, key is test_row_1/B:col10/1732765386394/Put/seqid=0 2024-11-28T03:43:07,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742381_1557 (size=9857) 2024-11-28T03:43:07,341 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/c75c7c78e7c74b619719afee20aa7977 2024-11-28T03:43:07,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/4d20d052eba94cf3a8504b0dd5f22b4c is 50, key is test_row_1/C:col10/1732765386394/Put/seqid=0 2024-11-28T03:43:07,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742382_1558 (size=9857) 2024-11-28T03:43:07,354 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/4d20d052eba94cf3a8504b0dd5f22b4c 2024-11-28T03:43:07,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/61c26fb5bfec400db1c370ef9c573b0d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/61c26fb5bfec400db1c370ef9c573b0d 2024-11-28T03:43:07,365 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/61c26fb5bfec400db1c370ef9c573b0d, entries=100, sequenceid=281, filesize=9.6 K 2024-11-28T03:43:07,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/c75c7c78e7c74b619719afee20aa7977 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c75c7c78e7c74b619719afee20aa7977 2024-11-28T03:43:07,368 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c75c7c78e7c74b619719afee20aa7977, entries=100, sequenceid=281, filesize=9.6 K 2024-11-28T03:43:07,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/4d20d052eba94cf3a8504b0dd5f22b4c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4d20d052eba94cf3a8504b0dd5f22b4c 2024-11-28T03:43:07,374 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4d20d052eba94cf3a8504b0dd5f22b4c, entries=100, sequenceid=281, filesize=9.6 K 2024-11-28T03:43:07,375 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=107.34 KB/109920 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 469ms, sequenceid=281, compaction requested=false 2024-11-28T03:43:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-28T03:43:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-28T03:43:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:07,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=114.05 KB heapSize=325.22 KB 2024-11-28T03:43:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:07,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-28T03:43:07,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 777 msec 2024-11-28T03:43:07,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 782 msec 2024-11-28T03:43:07,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/e6183ea84f3c40c78a12c78e735fe428 is 50, key is test_row_0/A:col10/1732765387357/Put/seqid=0 2024-11-28T03:43:07,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742383_1559 (size=14741) 2024-11-28T03:43:07,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765447495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:07,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765447607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:07,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765447646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:07,658 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:07,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765447656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:07,670 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:07,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765447667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:07,673 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_0' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:07,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765447676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:07,689 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T03:43:07,737 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:07,737 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-28T03:43:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T03:43:07,739 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:07,740 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:07,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:07,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/e6183ea84f3c40c78a12c78e735fe428 2024-11-28T03:43:07,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/c77dafac77064a03aa36f90727929cac is 50, key is test_row_0/B:col10/1732765387357/Put/seqid=0 2024-11-28T03:43:07,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742384_1560 (size=12301) 2024-11-28T03:43:07,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/c77dafac77064a03aa36f90727929cac 2024-11-28T03:43:07,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/2d845ab070724c7da9338609ad77c785 is 50, key is test_row_0/C:col10/1732765387357/Put/seqid=0 2024-11-28T03:43:07,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742385_1561 (size=12301) 2024-11-28T03:43:07,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765447826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T03:43:07,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-28T03:43:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:07,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:07,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:08,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-28T03:43:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:08,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:08,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:08,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:08,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T03:43:08,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765448136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:08,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-28T03:43:08,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:08,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:08,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:08,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:08,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:08,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/2d845ab070724c7da9338609ad77c785 2024-11-28T03:43:08,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/e6183ea84f3c40c78a12c78e735fe428 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/e6183ea84f3c40c78a12c78e735fe428 2024-11-28T03:43:08,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/e6183ea84f3c40c78a12c78e735fe428, entries=200, sequenceid=316, filesize=14.4 K 2024-11-28T03:43:08,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/c77dafac77064a03aa36f90727929cac as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c77dafac77064a03aa36f90727929cac 2024-11-28T03:43:08,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c77dafac77064a03aa36f90727929cac, entries=150, sequenceid=316, filesize=12.0 K 2024-11-28T03:43:08,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/2d845ab070724c7da9338609ad77c785 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2d845ab070724c7da9338609ad77c785 2024-11-28T03:43:08,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2d845ab070724c7da9338609ad77c785, entries=150, sequenceid=316, filesize=12.0 K 2024-11-28T03:43:08,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~325.17 KB/332976, currentSize=67.09 KB/68700 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 857ms, sequenceid=316, compaction requested=true 2024-11-28T03:43:08,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:08,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:08,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:08,234 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:08,234 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:08,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:08,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:08,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:08,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:08,235 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:08,235 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37547 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:08,235 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:43:08,235 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:43:08,235 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:08,235 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:08,235 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/32beaabbf4e34226a7348afaf3c74a9d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c75c7c78e7c74b619719afee20aa7977, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c77dafac77064a03aa36f90727929cac] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=34.3 K 2024-11-28T03:43:08,235 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/addf0d8df24a4800ae080dbb53411149, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/61c26fb5bfec400db1c370ef9c573b0d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/e6183ea84f3c40c78a12c78e735fe428] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=36.7 K 2024-11-28T03:43:08,235 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 32beaabbf4e34226a7348afaf3c74a9d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732765386206 2024-11-28T03:43:08,235 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting addf0d8df24a4800ae080dbb53411149, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732765386206 2024-11-28T03:43:08,236 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61c26fb5bfec400db1c370ef9c573b0d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732765386394 2024-11-28T03:43:08,236 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting c75c7c78e7c74b619719afee20aa7977, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732765386394 2024-11-28T03:43:08,236 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting c77dafac77064a03aa36f90727929cac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732765387341 2024-11-28T03:43:08,236 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6183ea84f3c40c78a12c78e735fe428, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732765387325 2024-11-28T03:43:08,245 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:08,245 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:08,245 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/1d4cda1bf2884b45a61e935c0789152f is 50, key is test_row_0/A:col10/1732765387357/Put/seqid=0 2024-11-28T03:43:08,245 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/6350db788e4548de8b831571b54a065c is 50, key is test_row_0/B:col10/1732765387357/Put/seqid=0 2024-11-28T03:43:08,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742386_1562 (size=13051) 2024-11-28T03:43:08,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742387_1563 (size=13051) 2024-11-28T03:43:08,263 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/6350db788e4548de8b831571b54a065c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6350db788e4548de8b831571b54a065c 2024-11-28T03:43:08,267 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 6350db788e4548de8b831571b54a065c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:08,267 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:08,267 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765388234; duration=0sec 2024-11-28T03:43:08,267 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:08,267 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:43:08,267 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:08,268 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:08,268 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:43:08,268 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:08,268 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6e96c8d855bb4351ad28134fdc98b01c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4d20d052eba94cf3a8504b0dd5f22b4c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2d845ab070724c7da9338609ad77c785] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=34.3 K 2024-11-28T03:43:08,269 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e96c8d855bb4351ad28134fdc98b01c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732765386206 2024-11-28T03:43:08,269 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d20d052eba94cf3a8504b0dd5f22b4c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732765386394 2024-11-28T03:43:08,269 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d845ab070724c7da9338609ad77c785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732765387341 2024-11-28T03:43:08,277 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:08,277 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/b78b3e0a6277447dbe77960ea9798143 is 50, key is test_row_0/C:col10/1732765387357/Put/seqid=0 2024-11-28T03:43:08,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742388_1564 (size=13051) 2024-11-28T03:43:08,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-28T03:43:08,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:08,352 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T03:43:08,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:08,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:08,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:08,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:08,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:08,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/469e5992d72b4f5b8f96254e28ab1ebe is 50, key is test_row_0/A:col10/1732765387438/Put/seqid=0 2024-11-28T03:43:08,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742389_1565 (size=12301) 2024-11-28T03:43:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T03:43:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:08,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:08,662 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/1d4cda1bf2884b45a61e935c0789152f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/1d4cda1bf2884b45a61e935c0789152f 2024-11-28T03:43:08,668 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 1d4cda1bf2884b45a61e935c0789152f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:08,668 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:08,668 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765388234; duration=0sec 2024-11-28T03:43:08,668 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:08,668 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:43:08,685 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/b78b3e0a6277447dbe77960ea9798143 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/b78b3e0a6277447dbe77960ea9798143 2024-11-28T03:43:08,690 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into b78b3e0a6277447dbe77960ea9798143(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:08,690 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:08,690 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765388234; duration=0sec 2024-11-28T03:43:08,690 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:08,690 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:43:08,760 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/469e5992d72b4f5b8f96254e28ab1ebe 2024-11-28T03:43:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/496351a491da4c3cbba2f9f9a06058b7 is 50, key is test_row_0/B:col10/1732765387438/Put/seqid=0 2024-11-28T03:43:08,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742390_1566 (size=12301) 2024-11-28T03:43:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765448773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T03:43:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765448886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:09,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765449096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:09,173 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/496351a491da4c3cbba2f9f9a06058b7 2024-11-28T03:43:09,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5a5655f8af9543b98709df91f9485f9d is 50, key is test_row_0/C:col10/1732765387438/Put/seqid=0 2024-11-28T03:43:09,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742391_1567 (size=12301) 2024-11-28T03:43:09,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765449416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:09,585 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5a5655f8af9543b98709df91f9485f9d 2024-11-28T03:43:09,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/469e5992d72b4f5b8f96254e28ab1ebe as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/469e5992d72b4f5b8f96254e28ab1ebe 2024-11-28T03:43:09,592 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/469e5992d72b4f5b8f96254e28ab1ebe, entries=150, sequenceid=330, filesize=12.0 K 2024-11-28T03:43:09,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/496351a491da4c3cbba2f9f9a06058b7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/496351a491da4c3cbba2f9f9a06058b7 2024-11-28T03:43:09,596 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/496351a491da4c3cbba2f9f9a06058b7, entries=150, sequenceid=330, filesize=12.0 K 2024-11-28T03:43:09,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5a5655f8af9543b98709df91f9485f9d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a5655f8af9543b98709df91f9485f9d 2024-11-28T03:43:09,599 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a5655f8af9543b98709df91f9485f9d, entries=150, sequenceid=330, filesize=12.0 K 2024-11-28T03:43:09,600 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 1248ms, sequenceid=330, compaction requested=false 2024-11-28T03:43:09,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:09,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:09,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-28T03:43:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-28T03:43:09,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-28T03:43:09,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8610 sec 2024-11-28T03:43:09,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.8670 sec 2024-11-28T03:43:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T03:43:09,887 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:09,887 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-28T03:43:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-28T03:43:09,889 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:09,890 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:09,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:09,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T03:43:09,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:09,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:09,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:09,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:09,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:09,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:09,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/cb0502b1274848f48dcf93d27461664e is 50, key is test_row_0/A:col10/1732765388761/Put/seqid=0 2024-11-28T03:43:09,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742392_1568 (size=14741) 2024-11-28T03:43:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-28T03:43:10,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765450026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:10,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-28T03:43:10,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:10,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:10,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:10,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:10,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:10,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765450146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:10,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:10,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:10,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:10,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-28T03:43:10,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:10,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:10,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:10,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:10,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-28T03:43:10,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-28T03:43:10,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:10,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:10,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:10,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:10,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:10,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/cb0502b1274848f48dcf93d27461664e 2024-11-28T03:43:10,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/b0f239e40338408aa0fd777f5f338b98 is 50, key is test_row_0/B:col10/1732765388761/Put/seqid=0 2024-11-28T03:43:10,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742393_1569 (size=12301) 2024-11-28T03:43:10,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/b0f239e40338408aa0fd777f5f338b98 2024-11-28T03:43:10,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8d8c23357d2046b78aaefe694984ea80 is 50, key is test_row_0/C:col10/1732765388761/Put/seqid=0 2024-11-28T03:43:10,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765450367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:10,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742394_1570 (size=12301) 2024-11-28T03:43:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:10,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8d8c23357d2046b78aaefe694984ea80 2024-11-28T03:43:10,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/cb0502b1274848f48dcf93d27461664e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/cb0502b1274848f48dcf93d27461664e 2024-11-28T03:43:10,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/cb0502b1274848f48dcf93d27461664e, entries=200, sequenceid=352, filesize=14.4 K 2024-11-28T03:43:10,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/b0f239e40338408aa0fd777f5f338b98 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b0f239e40338408aa0fd777f5f338b98 2024-11-28T03:43:10,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b0f239e40338408aa0fd777f5f338b98, entries=150, sequenceid=352, filesize=12.0 K 2024-11-28T03:43:10,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8d8c23357d2046b78aaefe694984ea80 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8d8c23357d2046b78aaefe694984ea80 2024-11-28T03:43:10,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8d8c23357d2046b78aaefe694984ea80, entries=150, sequenceid=352, filesize=12.0 K 2024-11-28T03:43:10,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 452ms, sequenceid=352, compaction requested=true 2024-11-28T03:43:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:10,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:10,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:10,395 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:10,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:10,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:10,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:10,395 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:10,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:10,395 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:10,395 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:10,396 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:43:10,396 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:43:10,396 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:10,396 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:10,396 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6350db788e4548de8b831571b54a065c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/496351a491da4c3cbba2f9f9a06058b7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b0f239e40338408aa0fd777f5f338b98] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=36.8 K 2024-11-28T03:43:10,396 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/1d4cda1bf2884b45a61e935c0789152f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/469e5992d72b4f5b8f96254e28ab1ebe, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/cb0502b1274848f48dcf93d27461664e] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=39.2 K 2024-11-28T03:43:10,396 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6350db788e4548de8b831571b54a065c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732765387341 2024-11-28T03:43:10,396 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d4cda1bf2884b45a61e935c0789152f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732765387341 2024-11-28T03:43:10,397 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 496351a491da4c3cbba2f9f9a06058b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732765387438 2024-11-28T03:43:10,397 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 469e5992d72b4f5b8f96254e28ab1ebe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732765387438 2024-11-28T03:43:10,399 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting b0f239e40338408aa0fd777f5f338b98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732765388761 2024-11-28T03:43:10,399 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb0502b1274848f48dcf93d27461664e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732765388754 2024-11-28T03:43:10,406 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:10,407 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/734429a3c084428baec1f35daa84c9d4 is 50, key is test_row_0/B:col10/1732765388761/Put/seqid=0 2024-11-28T03:43:10,407 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#493 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:10,408 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/76b835ffac614296a4c5f848f9cb5570 is 50, key is test_row_0/A:col10/1732765388761/Put/seqid=0 2024-11-28T03:43:10,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742395_1571 (size=13153) 2024-11-28T03:43:10,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742396_1572 (size=13153) 2024-11-28T03:43:10,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-28T03:43:10,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:10,503 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T03:43:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:10,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/fd016975d6194f0587a3ef6f79d14251 is 50, key is test_row_0/A:col10/1732765390008/Put/seqid=0 2024-11-28T03:43:10,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742397_1573 (size=12301) 2024-11-28T03:43:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-28T03:43:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:10,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:10,822 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/734429a3c084428baec1f35daa84c9d4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/734429a3c084428baec1f35daa84c9d4 2024-11-28T03:43:10,826 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 734429a3c084428baec1f35daa84c9d4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:10,826 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:10,826 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765390395; duration=0sec 2024-11-28T03:43:10,826 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:10,826 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:43:10,827 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:10,833 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:10,834 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:43:10,834 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:10,834 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/b78b3e0a6277447dbe77960ea9798143, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a5655f8af9543b98709df91f9485f9d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8d8c23357d2046b78aaefe694984ea80] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=36.8 K 2024-11-28T03:43:10,834 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting b78b3e0a6277447dbe77960ea9798143, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732765387341 2024-11-28T03:43:10,835 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a5655f8af9543b98709df91f9485f9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732765387438 2024-11-28T03:43:10,835 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d8c23357d2046b78aaefe694984ea80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732765388761 2024-11-28T03:43:10,836 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/76b835ffac614296a4c5f848f9cb5570 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/76b835ffac614296a4c5f848f9cb5570 2024-11-28T03:43:10,840 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 76b835ffac614296a4c5f848f9cb5570(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:10,840 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:10,840 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765390394; duration=0sec 2024-11-28T03:43:10,840 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:10,840 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:43:10,841 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:10,842 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/c01246644cc84172827448b96e69f54c is 50, key is test_row_0/C:col10/1732765388761/Put/seqid=0 2024-11-28T03:43:10,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742398_1574 (size=13153) 2024-11-28T03:43:10,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765450852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:10,912 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/fd016975d6194f0587a3ef6f79d14251 2024-11-28T03:43:10,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/23b2cc0070ea4830a448c87906d3de3f is 50, key is test_row_0/B:col10/1732765390008/Put/seqid=0 2024-11-28T03:43:10,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742399_1575 (size=12301) 2024-11-28T03:43:10,929 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/23b2cc0070ea4830a448c87906d3de3f 2024-11-28T03:43:10,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/9f0444f2f8a445a7b338d40a306317bd is 50, key is test_row_0/C:col10/1732765390008/Put/seqid=0 2024-11-28T03:43:10,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742400_1576 (size=12301) 2024-11-28T03:43:10,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765450976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:10,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:10,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-28T03:43:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765451196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:11,250 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/c01246644cc84172827448b96e69f54c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/c01246644cc84172827448b96e69f54c 2024-11-28T03:43:11,255 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into c01246644cc84172827448b96e69f54c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:11,255 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:11,255 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765390395; duration=0sec 2024-11-28T03:43:11,255 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:11,255 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:43:11,341 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/9f0444f2f8a445a7b338d40a306317bd 2024-11-28T03:43:11,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/fd016975d6194f0587a3ef6f79d14251 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/fd016975d6194f0587a3ef6f79d14251 2024-11-28T03:43:11,349 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/fd016975d6194f0587a3ef6f79d14251, entries=150, sequenceid=364, filesize=12.0 K 2024-11-28T03:43:11,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/23b2cc0070ea4830a448c87906d3de3f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/23b2cc0070ea4830a448c87906d3de3f 2024-11-28T03:43:11,352 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/23b2cc0070ea4830a448c87906d3de3f, entries=150, sequenceid=364, filesize=12.0 K 2024-11-28T03:43:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/9f0444f2f8a445a7b338d40a306317bd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/9f0444f2f8a445a7b338d40a306317bd 2024-11-28T03:43:11,356 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/9f0444f2f8a445a7b338d40a306317bd, entries=150, sequenceid=364, filesize=12.0 K 2024-11-28T03:43:11,357 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 855ms, sequenceid=364, compaction requested=false 2024-11-28T03:43:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-28T03:43:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-28T03:43:11,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-28T03:43:11,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4680 sec 2024-11-28T03:43:11,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 1.4730 sec 2024-11-28T03:43:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:11,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T03:43:11,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:11,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:11,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:11,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:11,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:11,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:11,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/526bc973c7944151a22e318b0036c711 is 50, key is test_row_0/A:col10/1732765391511/Put/seqid=0 2024-11-28T03:43:11,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742401_1577 (size=14741) 2024-11-28T03:43:11,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765451579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:11,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50604 deadline: 1732765451667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:11,674 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:11,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765451696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:11,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50598 deadline: 1732765451706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:11,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50576 deadline: 1732765451707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:11,712 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_0' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:11,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:11,713 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:11,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:11,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1732765451716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:11,723 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in e2c8ad458d2ec9e8bad5d996a8a6cef0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:11,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765451906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:11,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/526bc973c7944151a22e318b0036c711 2024-11-28T03:43:11,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/6aa8ae3f1cdd465d83107de816359336 is 50, key is test_row_0/B:col10/1732765391511/Put/seqid=0 2024-11-28T03:43:11,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742402_1578 (size=12301) 2024-11-28T03:43:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-28T03:43:12,037 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:12,037 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees 2024-11-28T03:43:12,039 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-28T03:43:12,040 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:12,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-28T03:43:12,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-28T03:43:12,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:12,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:12,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:12,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:12,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:12,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765452226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:12,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/6aa8ae3f1cdd465d83107de816359336 2024-11-28T03:43:12,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/0da186b58d5745c6907e829b2c464cb4 is 50, key is test_row_0/C:col10/1732765391511/Put/seqid=0 2024-11-28T03:43:12,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-28T03:43:12,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:12,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:12,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:12,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:12,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:12,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742403_1579 (size=12301) 2024-11-28T03:43:12,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/0da186b58d5745c6907e829b2c464cb4 2024-11-28T03:43:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-28T03:43:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/526bc973c7944151a22e318b0036c711 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/526bc973c7944151a22e318b0036c711 2024-11-28T03:43:12,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/526bc973c7944151a22e318b0036c711, entries=200, sequenceid=388, filesize=14.4 K 2024-11-28T03:43:12,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/6aa8ae3f1cdd465d83107de816359336 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6aa8ae3f1cdd465d83107de816359336 2024-11-28T03:43:12,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6aa8ae3f1cdd465d83107de816359336, entries=150, sequenceid=388, filesize=12.0 K 2024-11-28T03:43:12,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/0da186b58d5745c6907e829b2c464cb4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/0da186b58d5745c6907e829b2c464cb4 2024-11-28T03:43:12,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/0da186b58d5745c6907e829b2c464cb4, entries=150, sequenceid=388, filesize=12.0 K 2024-11-28T03:43:12,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 866ms, sequenceid=388, compaction requested=true 2024-11-28T03:43:12,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:12,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:12,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:12,378 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:12,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:12,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:12,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:12,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T03:43:12,378 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:12,380 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:12,380 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:12,380 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:43:12,380 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:43:12,380 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:12,380 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:12,380 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/734429a3c084428baec1f35daa84c9d4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/23b2cc0070ea4830a448c87906d3de3f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6aa8ae3f1cdd465d83107de816359336] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=36.9 K 2024-11-28T03:43:12,380 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/76b835ffac614296a4c5f848f9cb5570, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/fd016975d6194f0587a3ef6f79d14251, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/526bc973c7944151a22e318b0036c711] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=39.3 K 2024-11-28T03:43:12,380 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b835ffac614296a4c5f848f9cb5570, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732765388761 2024-11-28T03:43:12,380 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 734429a3c084428baec1f35daa84c9d4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732765388761 2024-11-28T03:43:12,380 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd016975d6194f0587a3ef6f79d14251, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732765390008 2024-11-28T03:43:12,381 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b2cc0070ea4830a448c87906d3de3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732765390008 2024-11-28T03:43:12,381 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 526bc973c7944151a22e318b0036c711, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732765390793 2024-11-28T03:43:12,381 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aa8ae3f1cdd465d83107de816359336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732765390804 2024-11-28T03:43:12,390 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:12,391 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/52b1edc7989442c8b4b9169fd5258f74 is 50, key is test_row_0/A:col10/1732765391511/Put/seqid=0 2024-11-28T03:43:12,394 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:12,394 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/02a12ca28d3b4264985dc571bc298ac9 is 50, key is test_row_0/B:col10/1732765391511/Put/seqid=0 2024-11-28T03:43:12,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742404_1580 (size=13255) 2024-11-28T03:43:12,415 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/52b1edc7989442c8b4b9169fd5258f74 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52b1edc7989442c8b4b9169fd5258f74 2024-11-28T03:43:12,420 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 52b1edc7989442c8b4b9169fd5258f74(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:12,420 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:12,421 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765392378; duration=0sec 2024-11-28T03:43:12,421 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:12,421 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:43:12,421 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:12,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742405_1581 (size=13255) 2024-11-28T03:43:12,425 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:12,425 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:43:12,425 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:12,425 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/c01246644cc84172827448b96e69f54c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/9f0444f2f8a445a7b338d40a306317bd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/0da186b58d5745c6907e829b2c464cb4] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=36.9 K 2024-11-28T03:43:12,425 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c01246644cc84172827448b96e69f54c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732765388761 2024-11-28T03:43:12,426 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f0444f2f8a445a7b338d40a306317bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732765390008 2024-11-28T03:43:12,426 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da186b58d5745c6907e829b2c464cb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732765390804 2024-11-28T03:43:12,435 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:12,436 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/46e31fad10394d908c971e9d07e3b1c7 is 50, key is test_row_0/C:col10/1732765391511/Put/seqid=0 2024-11-28T03:43:12,437 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/02a12ca28d3b4264985dc571bc298ac9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/02a12ca28d3b4264985dc571bc298ac9 2024-11-28T03:43:12,447 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 02a12ca28d3b4264985dc571bc298ac9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:12,447 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:12,447 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765392378; duration=0sec 2024-11-28T03:43:12,447 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:12,447 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:43:12,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742406_1582 (size=13255) 2024-11-28T03:43:12,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-28T03:43:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:12,500 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T03:43:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/d5912243e08243f0a10f45e8081d0852 is 50, key is test_row_1/A:col10/1732765391574/Put/seqid=0 2024-11-28T03:43:12,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742407_1583 (size=9857) 2024-11-28T03:43:12,515 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/d5912243e08243f0a10f45e8081d0852 2024-11-28T03:43:12,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/5967c1a6beea4b61a689bfc653a73eae is 50, key is test_row_1/B:col10/1732765391574/Put/seqid=0 2024-11-28T03:43:12,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742408_1584 (size=9857) 2024-11-28T03:43:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-28T03:43:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:12,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:12,852 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/46e31fad10394d908c971e9d07e3b1c7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/46e31fad10394d908c971e9d07e3b1c7 2024-11-28T03:43:12,855 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 46e31fad10394d908c971e9d07e3b1c7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:12,855 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:12,855 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765392378; duration=0sec 2024-11-28T03:43:12,856 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:12,856 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:43:12,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765392844, max timestamp=1732765392879Num uniques 859; 2024-11-28T03:43:12,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:43:12,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:12,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765452919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:12,936 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/5967c1a6beea4b61a689bfc653a73eae 2024-11-28T03:43:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/42b033fdf9c0401c8329471392f5a18f is 50, key is test_row_1/C:col10/1732765391574/Put/seqid=0 2024-11-28T03:43:12,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742409_1585 (size=9857) 2024-11-28T03:43:13,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765453036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-28T03:43:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765453246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:13,346 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/42b033fdf9c0401c8329471392f5a18f 2024-11-28T03:43:13,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/d5912243e08243f0a10f45e8081d0852 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d5912243e08243f0a10f45e8081d0852 2024-11-28T03:43:13,358 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d5912243e08243f0a10f45e8081d0852, entries=100, sequenceid=401, filesize=9.6 K 2024-11-28T03:43:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/5967c1a6beea4b61a689bfc653a73eae as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/5967c1a6beea4b61a689bfc653a73eae 2024-11-28T03:43:13,363 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/5967c1a6beea4b61a689bfc653a73eae, entries=100, sequenceid=401, filesize=9.6 K 2024-11-28T03:43:13,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/42b033fdf9c0401c8329471392f5a18f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/42b033fdf9c0401c8329471392f5a18f 2024-11-28T03:43:13,366 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/42b033fdf9c0401c8329471392f5a18f, entries=100, sequenceid=401, filesize=9.6 K 2024-11-28T03:43:13,367 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 867ms, sequenceid=401, compaction requested=false 2024-11-28T03:43:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=148 2024-11-28T03:43:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=148 2024-11-28T03:43:13,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-28T03:43:13,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3290 sec 2024-11-28T03:43:13,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees in 1.3340 sec 2024-11-28T03:43:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:13,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=131.94 KB heapSize=386.72 KB 2024-11-28T03:43:13,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:13,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:13,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:13,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:13,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:13,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:13,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/7022fd289ab243d4807bffe7f7e286e9 is 50, key is test_row_0/A:col10/1732765393573/Put/seqid=0 2024-11-28T03:43:13,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742410_1586 (size=14741) 2024-11-28T03:43:13,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765453633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:13,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765453746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:13,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765453966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:13,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/7022fd289ab243d4807bffe7f7e286e9 2024-11-28T03:43:13,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/74b889f1e4d74506a5978fb34a562196 is 50, key is test_row_0/B:col10/1732765393573/Put/seqid=0 2024-11-28T03:43:13,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742411_1587 (size=12301) 2024-11-28T03:43:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-28T03:43:14,187 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:14,188 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees 2024-11-28T03:43:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-28T03:43:14,190 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:14,190 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:14,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:14,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765454277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-28T03:43:14,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-28T03:43:14,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:14,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:14,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:14,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:14,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:14,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/74b889f1e4d74506a5978fb34a562196 2024-11-28T03:43:14,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8c3f29ebc2af40fab4fcb7c607edf144 is 50, key is test_row_0/C:col10/1732765393573/Put/seqid=0 2024-11-28T03:43:14,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742412_1588 (size=12301) 2024-11-28T03:43:14,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8c3f29ebc2af40fab4fcb7c607edf144 2024-11-28T03:43:14,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/7022fd289ab243d4807bffe7f7e286e9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7022fd289ab243d4807bffe7f7e286e9 2024-11-28T03:43:14,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7022fd289ab243d4807bffe7f7e286e9, entries=200, sequenceid=429, filesize=14.4 K 2024-11-28T03:43:14,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/74b889f1e4d74506a5978fb34a562196 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/74b889f1e4d74506a5978fb34a562196 2024-11-28T03:43:14,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/74b889f1e4d74506a5978fb34a562196, entries=150, sequenceid=429, filesize=12.0 K 2024-11-28T03:43:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8c3f29ebc2af40fab4fcb7c607edf144 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c3f29ebc2af40fab4fcb7c607edf144 2024-11-28T03:43:14,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c3f29ebc2af40fab4fcb7c607edf144, entries=150, sequenceid=429, filesize=12.0 K 2024-11-28T03:43:14,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~386.67 KB/395952, currentSize=46.96 KB/48090 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 878ms, sequenceid=429, compaction requested=true 2024-11-28T03:43:14,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:14,453 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:14,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:14,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:14,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:14,453 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:14,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:14,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:14,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:14,454 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37853 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:14,454 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:43:14,454 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:14,454 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52b1edc7989442c8b4b9169fd5258f74, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d5912243e08243f0a10f45e8081d0852, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7022fd289ab243d4807bffe7f7e286e9] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=37.0 K 2024-11-28T03:43:14,454 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:14,454 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:43:14,454 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:14,454 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/02a12ca28d3b4264985dc571bc298ac9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/5967c1a6beea4b61a689bfc653a73eae, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/74b889f1e4d74506a5978fb34a562196] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=34.6 K 2024-11-28T03:43:14,454 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b1edc7989442c8b4b9169fd5258f74, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732765390804 2024-11-28T03:43:14,454 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 02a12ca28d3b4264985dc571bc298ac9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732765390804 2024-11-28T03:43:14,455 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5912243e08243f0a10f45e8081d0852, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732765391574 2024-11-28T03:43:14,455 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5967c1a6beea4b61a689bfc653a73eae, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732765391574 2024-11-28T03:43:14,455 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7022fd289ab243d4807bffe7f7e286e9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732765392869 2024-11-28T03:43:14,455 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 74b889f1e4d74506a5978fb34a562196, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732765392869 2024-11-28T03:43:14,461 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:14,461 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/37dbb990318143b18109efb7fe81d311 is 50, key is test_row_0/B:col10/1732765393573/Put/seqid=0 2024-11-28T03:43:14,463 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#511 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:14,463 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/2bc596e08e8149d7892169043cbece33 is 50, key is test_row_0/A:col10/1732765393573/Put/seqid=0 2024-11-28T03:43:14,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742414_1590 (size=13357) 2024-11-28T03:43:14,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742413_1589 (size=13357) 2024-11-28T03:43:14,486 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/37dbb990318143b18109efb7fe81d311 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/37dbb990318143b18109efb7fe81d311 2024-11-28T03:43:14,487 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/2bc596e08e8149d7892169043cbece33 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2bc596e08e8149d7892169043cbece33 2024-11-28T03:43:14,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-28T03:43:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:14,497 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:43:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:14,500 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 2bc596e08e8149d7892169043cbece33(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:14,500 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:14,500 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765394453; duration=0sec 2024-11-28T03:43:14,500 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:14,500 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:43:14,500 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:14,501 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 37dbb990318143b18109efb7fe81d311(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:14,501 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:14,501 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765394453; duration=0sec 2024-11-28T03:43:14,501 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:14,501 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:43:14,505 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:14,505 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:43:14,505 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:14,505 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/46e31fad10394d908c971e9d07e3b1c7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/42b033fdf9c0401c8329471392f5a18f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c3f29ebc2af40fab4fcb7c607edf144] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=34.6 K 2024-11-28T03:43:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-28T03:43:14,508 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e31fad10394d908c971e9d07e3b1c7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732765390804 2024-11-28T03:43:14,510 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b033fdf9c0401c8329471392f5a18f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732765391574 2024-11-28T03:43:14,511 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c3f29ebc2af40fab4fcb7c607edf144, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732765392869 2024-11-28T03:43:14,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/d0ceeef721db423ea16e15e53eb86f13 is 50, key is test_row_0/A:col10/1732765393603/Put/seqid=0 2024-11-28T03:43:14,548 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:14,549 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/ea4d9a753c174bd4b735397751a6937d is 50, key is test_row_0/C:col10/1732765393573/Put/seqid=0 2024-11-28T03:43:14,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742415_1591 (size=12301) 2024-11-28T03:43:14,572 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/d0ceeef721db423ea16e15e53eb86f13 2024-11-28T03:43:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/d408621c06be4444869b1d00ceebe111 is 50, key is test_row_0/B:col10/1732765393603/Put/seqid=0 2024-11-28T03:43:14,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742416_1592 (size=13357) 2024-11-28T03:43:14,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742417_1593 (size=12301) 2024-11-28T03:43:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:14,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-28T03:43:15,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765394934, max timestamp=1732765395017Num uniques 859; 2024-11-28T03:43:15,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:15,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765394934, max timestamp=1732765395017Num uniques 859; 2024-11-28T03:43:15,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:15,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732765394934, max timestamp=1732765395017Num uniques 859; 2024-11-28T03:43:15,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T03:43:15,048 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/ea4d9a753c174bd4b735397751a6937d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/ea4d9a753c174bd4b735397751a6937d 2024-11-28T03:43:15,056 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into ea4d9a753c174bd4b735397751a6937d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:15,056 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:15,056 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765394453; duration=0sec 2024-11-28T03:43:15,056 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:15,056 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:43:15,061 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/d408621c06be4444869b1d00ceebe111 2024-11-28T03:43:15,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/1b8cc1e8952c4196936d9f9fa18a19d2 is 50, key is test_row_0/C:col10/1732765393603/Put/seqid=0 2024-11-28T03:43:15,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742418_1594 (size=12301) 2024-11-28T03:43:15,074 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/1b8cc1e8952c4196936d9f9fa18a19d2 2024-11-28T03:43:15,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/d0ceeef721db423ea16e15e53eb86f13 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d0ceeef721db423ea16e15e53eb86f13 2024-11-28T03:43:15,088 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d0ceeef721db423ea16e15e53eb86f13, entries=150, sequenceid=440, filesize=12.0 K 2024-11-28T03:43:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/d408621c06be4444869b1d00ceebe111 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d408621c06be4444869b1d00ceebe111 2024-11-28T03:43:15,094 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d408621c06be4444869b1d00ceebe111, entries=150, sequenceid=440, filesize=12.0 K 2024-11-28T03:43:15,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/1b8cc1e8952c4196936d9f9fa18a19d2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/1b8cc1e8952c4196936d9f9fa18a19d2 2024-11-28T03:43:15,098 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/1b8cc1e8952c4196936d9f9fa18a19d2, entries=150, sequenceid=440, filesize=12.0 K 2024-11-28T03:43:15,099 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=67.09 KB/68700 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 602ms, sequenceid=440, compaction requested=false 2024-11-28T03:43:15,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:15,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:15,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=150 2024-11-28T03:43:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=150 2024-11-28T03:43:15,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-28T03:43:15,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 909 msec 2024-11-28T03:43:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:15,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=73.80 KB heapSize=205.69 KB 2024-11-28T03:43:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:15,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees in 916 msec 2024-11-28T03:43:15,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/9436b825208347988d9f9ef0c03d2d5c is 50, key is test_row_0/A:col10/1732765395039/Put/seqid=0 2024-11-28T03:43:15,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742419_1595 (size=14741) 2024-11-28T03:43:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-28T03:43:15,326 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:15,327 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees 2024-11-28T03:43:15,332 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:15,333 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-28T03:43:15,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:15,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765455313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-28T03:43:15,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765455447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:15,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-28T03:43:15,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:15,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:15,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:15,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:15,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/9436b825208347988d9f9ef0c03d2d5c 2024-11-28T03:43:15,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/ea103568c43a4644b360a954e7ac3e25 is 50, key is test_row_0/B:col10/1732765395039/Put/seqid=0 2024-11-28T03:43:15,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742420_1596 (size=12301) 2024-11-28T03:43:15,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/ea103568c43a4644b360a954e7ac3e25 2024-11-28T03:43:15,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5a16968c88524089b37c09732344355f is 50, key is test_row_0/C:col10/1732765395039/Put/seqid=0 2024-11-28T03:43:15,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742421_1597 (size=12301) 2024-11-28T03:43:15,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5a16968c88524089b37c09732344355f 2024-11-28T03:43:15,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/9436b825208347988d9f9ef0c03d2d5c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9436b825208347988d9f9ef0c03d2d5c 2024-11-28T03:43:15,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9436b825208347988d9f9ef0c03d2d5c, entries=200, sequenceid=470, filesize=14.4 K 2024-11-28T03:43:15,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/ea103568c43a4644b360a954e7ac3e25 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ea103568c43a4644b360a954e7ac3e25 2024-11-28T03:43:15,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ea103568c43a4644b360a954e7ac3e25, entries=150, sequenceid=470, filesize=12.0 K 2024-11-28T03:43:15,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/5a16968c88524089b37c09732344355f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a16968c88524089b37c09732344355f 2024-11-28T03:43:15,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a16968c88524089b37c09732344355f, entries=150, sequenceid=470, filesize=12.0 K 2024-11-28T03:43:15,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.64 KB/210576, currentSize=107.34 KB/109920 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 502ms, sequenceid=470, compaction requested=true 2024-11-28T03:43:15,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:15,604 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:15,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:15,605 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:15,605 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:43:15,605 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:15,605 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2bc596e08e8149d7892169043cbece33, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d0ceeef721db423ea16e15e53eb86f13, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9436b825208347988d9f9ef0c03d2d5c] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=39.5 K 2024-11-28T03:43:15,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:15,605 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:15,605 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bc596e08e8149d7892169043cbece33, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732765392869 2024-11-28T03:43:15,606 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0ceeef721db423ea16e15e53eb86f13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732765393603 2024-11-28T03:43:15,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:15,606 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9436b825208347988d9f9ef0c03d2d5c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732765395039 2024-11-28T03:43:15,606 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:15,607 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:43:15,607 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:15,607 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/37dbb990318143b18109efb7fe81d311, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d408621c06be4444869b1d00ceebe111, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ea103568c43a4644b360a954e7ac3e25] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=37.1 K 2024-11-28T03:43:15,607 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 37dbb990318143b18109efb7fe81d311, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732765392869 2024-11-28T03:43:15,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:15,608 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d408621c06be4444869b1d00ceebe111, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732765393603 2024-11-28T03:43:15,611 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ea103568c43a4644b360a954e7ac3e25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732765395039 2024-11-28T03:43:15,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:15,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:15,628 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#519 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:15,629 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/f482d86613a94dd4a1e7b1a0bb28c4d6 is 50, key is test_row_0/A:col10/1732765395039/Put/seqid=0 2024-11-28T03:43:15,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-28T03:43:15,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:15,640 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:43:15,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:15,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:15,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:15,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:15,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:15,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:15,646 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#520 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:15,646 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/4309fd119168443d84a04f2a3fef1e58 is 50, key is test_row_0/B:col10/1732765395039/Put/seqid=0 2024-11-28T03:43:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-28T03:43:15,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/60be5800dbf14c4eab7e87190cf19e5c is 50, key is test_row_0/A:col10/1732765395262/Put/seqid=0 2024-11-28T03:43:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:15,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:15,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742422_1598 (size=13459) 2024-11-28T03:43:15,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742423_1599 (size=13459) 2024-11-28T03:43:15,717 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/4309fd119168443d84a04f2a3fef1e58 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4309fd119168443d84a04f2a3fef1e58 2024-11-28T03:43:15,722 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 4309fd119168443d84a04f2a3fef1e58(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:15,722 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:15,722 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765395605; duration=0sec 2024-11-28T03:43:15,722 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:15,722 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:43:15,722 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:15,723 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:15,723 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:43:15,723 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:15,723 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/ea4d9a753c174bd4b735397751a6937d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/1b8cc1e8952c4196936d9f9fa18a19d2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a16968c88524089b37c09732344355f] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=37.1 K 2024-11-28T03:43:15,724 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting ea4d9a753c174bd4b735397751a6937d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732765392869 2024-11-28T03:43:15,725 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8cc1e8952c4196936d9f9fa18a19d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732765393603 2024-11-28T03:43:15,725 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a16968c88524089b37c09732344355f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732765395039 2024-11-28T03:43:15,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742424_1600 (size=12301) 2024-11-28T03:43:15,729 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/60be5800dbf14c4eab7e87190cf19e5c 2024-11-28T03:43:15,737 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#522 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:15,737 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/155bc0d4ecd44697b6af7d36f53d87f6 is 50, key is test_row_0/C:col10/1732765395039/Put/seqid=0 2024-11-28T03:43:15,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/b1b0e01d4ba147f1ba8df40b06726411 is 50, key is test_row_0/B:col10/1732765395262/Put/seqid=0 2024-11-28T03:43:15,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742425_1601 (size=13459) 2024-11-28T03:43:15,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742426_1602 (size=12301) 2024-11-28T03:43:15,755 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/b1b0e01d4ba147f1ba8df40b06726411 2024-11-28T03:43:15,760 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/155bc0d4ecd44697b6af7d36f53d87f6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/155bc0d4ecd44697b6af7d36f53d87f6 2024-11-28T03:43:15,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/df7da848638f466dad271e77d9710d96 is 50, key is test_row_0/C:col10/1732765395262/Put/seqid=0 2024-11-28T03:43:15,773 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 155bc0d4ecd44697b6af7d36f53d87f6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:15,773 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:15,774 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765395610; duration=0sec 2024-11-28T03:43:15,774 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:15,774 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:43:15,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742427_1603 (size=12301) 2024-11-28T03:43:15,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765455801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:15,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765455916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-28T03:43:16,113 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/f482d86613a94dd4a1e7b1a0bb28c4d6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/f482d86613a94dd4a1e7b1a0bb28c4d6 2024-11-28T03:43:16,122 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into f482d86613a94dd4a1e7b1a0bb28c4d6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:16,122 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:16,122 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765395604; duration=0sec 2024-11-28T03:43:16,122 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:16,122 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:43:16,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765456127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:16,186 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/df7da848638f466dad271e77d9710d96 2024-11-28T03:43:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/60be5800dbf14c4eab7e87190cf19e5c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/60be5800dbf14c4eab7e87190cf19e5c 2024-11-28T03:43:16,199 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/60be5800dbf14c4eab7e87190cf19e5c, entries=150, sequenceid=489, filesize=12.0 K 2024-11-28T03:43:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/b1b0e01d4ba147f1ba8df40b06726411 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b1b0e01d4ba147f1ba8df40b06726411 2024-11-28T03:43:16,204 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b1b0e01d4ba147f1ba8df40b06726411, entries=150, sequenceid=489, filesize=12.0 K 2024-11-28T03:43:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/df7da848638f466dad271e77d9710d96 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/df7da848638f466dad271e77d9710d96 2024-11-28T03:43:16,210 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/df7da848638f466dad271e77d9710d96, entries=150, sequenceid=489, filesize=12.0 K 2024-11-28T03:43:16,211 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 570ms, sequenceid=489, compaction requested=false 2024-11-28T03:43:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-11-28T03:43:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=152 2024-11-28T03:43:16,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-28T03:43:16,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 880 msec 2024-11-28T03:43:16,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees in 891 msec 2024-11-28T03:43:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:16,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:43:16,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:16,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:16,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:16,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:16,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:16,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:16,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/45b2fefa53f944aa9e6a1a4cb1e06243 is 50, key is test_row_0/A:col10/1732765396439/Put/seqid=0 2024-11-28T03:43:16,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742428_1604 (size=14741) 2024-11-28T03:43:16,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/45b2fefa53f944aa9e6a1a4cb1e06243 2024-11-28T03:43:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-28T03:43:16,468 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:16,469 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees 2024-11-28T03:43:16,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/80e39915d04943798865dc00b86d481f is 50, key is test_row_0/B:col10/1732765396439/Put/seqid=0 2024-11-28T03:43:16,470 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=153, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-28T03:43:16,471 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=153, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:16,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:16,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742429_1605 (size=12301) 2024-11-28T03:43:16,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/80e39915d04943798865dc00b86d481f 2024-11-28T03:43:16,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/06c52439e7f344a9b1c491a6dfb34188 is 50, key is test_row_0/C:col10/1732765396439/Put/seqid=0 2024-11-28T03:43:16,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742430_1606 (size=12301) 2024-11-28T03:43:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-28T03:43:16,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765456592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:16,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=154 2024-11-28T03:43:16,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:16,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:16,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:16,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] handler.RSProcedureHandler(58): pid=154 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:16,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=154 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=154 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:16,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732765456717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 , the old value is region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., hostname=3a546af21488,34671,1732765249447, seqNum=2 because the exception is null or not the one we care about 2024-11-28T03:43:16,740 INFO [Thread-2214 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2214. 2024-11-28T03:43:16,740 DEBUG [Thread-2214 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:16,740 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,740 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:16,743 INFO [Thread-2216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2216. 2024-11-28T03:43:16,743 DEBUG [Thread-2216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:16,743 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,744 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:16,746 INFO [Thread-2218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2218. 2024-11-28T03:43:16,746 DEBUG [Thread-2218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:16,746 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,747 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,747 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:16,753 INFO [Thread-2210 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2210. 2024-11-28T03:43:16,753 DEBUG [Thread-2210 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:16,753 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,753 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:16,754 INFO [Thread-2212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2212. 2024-11-28T03:43:16,754 DEBUG [Thread-2212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:16,754 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,754 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:16,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=154 2024-11-28T03:43:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:16,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=154}] handler.RSProcedureHandler(58): pid=154 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=154 java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=154 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-28T03:43:16,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/06c52439e7f344a9b1c491a6dfb34188 2024-11-28T03:43:16,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/45b2fefa53f944aa9e6a1a4cb1e06243 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/45b2fefa53f944aa9e6a1a4cb1e06243 2024-11-28T03:43:16,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/45b2fefa53f944aa9e6a1a4cb1e06243, entries=200, sequenceid=506, filesize=14.4 K 2024-11-28T03:43:16,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/80e39915d04943798865dc00b86d481f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/80e39915d04943798865dc00b86d481f 2024-11-28T03:43:16,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/80e39915d04943798865dc00b86d481f, entries=150, sequenceid=506, filesize=12.0 K 2024-11-28T03:43:16,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/06c52439e7f344a9b1c491a6dfb34188 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/06c52439e7f344a9b1c491a6dfb34188 2024-11-28T03:43:16,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/06c52439e7f344a9b1c491a6dfb34188, entries=150, sequenceid=506, filesize=12.0 K 2024-11-28T03:43:16,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 473ms, sequenceid=506, compaction requested=true 2024-11-28T03:43:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:16,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:16,914 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:16,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:16,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2c8ad458d2ec9e8bad5d996a8a6cef0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:16,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:16,914 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:16,914 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:16,914 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/A is initiating minor compaction (all files) 2024-11-28T03:43:16,914 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:16,914 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/A in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:16,914 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/B is initiating minor compaction (all files) 2024-11-28T03:43:16,914 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/f482d86613a94dd4a1e7b1a0bb28c4d6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/60be5800dbf14c4eab7e87190cf19e5c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/45b2fefa53f944aa9e6a1a4cb1e06243] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=39.6 K 2024-11-28T03:43:16,914 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/B in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:16,914 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4309fd119168443d84a04f2a3fef1e58, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b1b0e01d4ba147f1ba8df40b06726411, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/80e39915d04943798865dc00b86d481f] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=37.2 K 2024-11-28T03:43:16,915 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f482d86613a94dd4a1e7b1a0bb28c4d6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732765395039 2024-11-28T03:43:16,915 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4309fd119168443d84a04f2a3fef1e58, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732765395039 2024-11-28T03:43:16,915 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60be5800dbf14c4eab7e87190cf19e5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732765395262 2024-11-28T03:43:16,915 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b0e01d4ba147f1ba8df40b06726411, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732765395262 2024-11-28T03:43:16,915 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45b2fefa53f944aa9e6a1a4cb1e06243, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732765395702 2024-11-28T03:43:16,915 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 80e39915d04943798865dc00b86d481f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732765395702 2024-11-28T03:43:16,920 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#B#compaction#528 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:16,920 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#A#compaction#529 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:16,921 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/31bccf942fcb45e0aa0f77859dd33bc4 is 50, key is test_row_0/B:col10/1732765396439/Put/seqid=0 2024-11-28T03:43:16,921 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/c1bcc3f05d8241e8bee5e707dad576b6 is 50, key is test_row_0/A:col10/1732765396439/Put/seqid=0 2024-11-28T03:43:16,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742431_1607 (size=13561) 2024-11-28T03:43:16,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742432_1608 (size=13561) 2024-11-28T03:43:16,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=154 2024-11-28T03:43:16,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:16,931 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:43:16,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:16,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:16,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:16,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/e0bfb6e853534c49a35e9e99c621b5fe is 50, key is test_row_0/A:col10/1732765396557/Put/seqid=0 2024-11-28T03:43:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:16,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. as already flushing 2024-11-28T03:43:16,938 INFO [Thread-2203 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2203. 2024-11-28T03:43:16,938 DEBUG [Thread-2203 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:16,938 DEBUG [Thread-2203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,938 DEBUG [Thread-2203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:16,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:16,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742433_1609 (size=12301) 2024-11-28T03:43:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-28T03:43:17,329 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/31bccf942fcb45e0aa0f77859dd33bc4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/31bccf942fcb45e0aa0f77859dd33bc4 2024-11-28T03:43:17,329 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/c1bcc3f05d8241e8bee5e707dad576b6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c1bcc3f05d8241e8bee5e707dad576b6 2024-11-28T03:43:17,335 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/B of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 31bccf942fcb45e0aa0f77859dd33bc4(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:17,335 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/A of e2c8ad458d2ec9e8bad5d996a8a6cef0 into c1bcc3f05d8241e8bee5e707dad576b6(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:17,335 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:17,335 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:17,335 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/B, priority=13, startTime=1732765396913; duration=0sec 2024-11-28T03:43:17,335 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/A, priority=13, startTime=1732765396913; duration=0sec 2024-11-28T03:43:17,335 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:17,335 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:17,335 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:A 2024-11-28T03:43:17,335 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:B 2024-11-28T03:43:17,335 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:17,336 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:17,336 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): e2c8ad458d2ec9e8bad5d996a8a6cef0/C is initiating minor compaction (all files) 2024-11-28T03:43:17,336 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c8ad458d2ec9e8bad5d996a8a6cef0/C in TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:17,336 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/155bc0d4ecd44697b6af7d36f53d87f6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/df7da848638f466dad271e77d9710d96, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/06c52439e7f344a9b1c491a6dfb34188] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp, totalSize=37.2 K 2024-11-28T03:43:17,337 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 155bc0d4ecd44697b6af7d36f53d87f6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732765395039 2024-11-28T03:43:17,337 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting df7da848638f466dad271e77d9710d96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732765395262 2024-11-28T03:43:17,337 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06c52439e7f344a9b1c491a6dfb34188, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732765395702 2024-11-28T03:43:17,342 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/e0bfb6e853534c49a35e9e99c621b5fe 2024-11-28T03:43:17,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/1ab965d3b32245a7a023ce4cfd39c790 is 50, key is test_row_0/B:col10/1732765396557/Put/seqid=0 2024-11-28T03:43:17,350 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c8ad458d2ec9e8bad5d996a8a6cef0#C#compaction#532 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:17,350 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8d4916840e7b4564b24c777bbaaae8bb is 50, key is test_row_0/C:col10/1732765396439/Put/seqid=0 2024-11-28T03:43:17,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742434_1610 (size=12301) 2024-11-28T03:43:17,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742435_1611 (size=13561) 2024-11-28T03:43:17,605 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T03:43:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-28T03:43:17,753 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/1ab965d3b32245a7a023ce4cfd39c790 2024-11-28T03:43:17,761 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/8d4916840e7b4564b24c777bbaaae8bb as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8d4916840e7b4564b24c777bbaaae8bb 2024-11-28T03:43:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/270b698a7b60446f94b4440672618c5d is 50, key is test_row_0/C:col10/1732765396557/Put/seqid=0 2024-11-28T03:43:17,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742436_1612 (size=12301) 2024-11-28T03:43:17,765 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2c8ad458d2ec9e8bad5d996a8a6cef0/C of e2c8ad458d2ec9e8bad5d996a8a6cef0 into 8d4916840e7b4564b24c777bbaaae8bb(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:17,765 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:17,765 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., storeName=e2c8ad458d2ec9e8bad5d996a8a6cef0/C, priority=13, startTime=1732765396914; duration=0sec 2024-11-28T03:43:17,765 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:17,765 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c8ad458d2ec9e8bad5d996a8a6cef0:C 2024-11-28T03:43:18,165 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/270b698a7b60446f94b4440672618c5d 2024-11-28T03:43:18,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/e0bfb6e853534c49a35e9e99c621b5fe as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/e0bfb6e853534c49a35e9e99c621b5fe 2024-11-28T03:43:18,172 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/e0bfb6e853534c49a35e9e99c621b5fe, entries=150, sequenceid=524, filesize=12.0 K 2024-11-28T03:43:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/1ab965d3b32245a7a023ce4cfd39c790 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/1ab965d3b32245a7a023ce4cfd39c790 2024-11-28T03:43:18,175 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/1ab965d3b32245a7a023ce4cfd39c790, entries=150, sequenceid=524, filesize=12.0 K 2024-11-28T03:43:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/270b698a7b60446f94b4440672618c5d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/270b698a7b60446f94b4440672618c5d 2024-11-28T03:43:18,179 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/270b698a7b60446f94b4440672618c5d, entries=150, sequenceid=524, filesize=12.0 K 2024-11-28T03:43:18,180 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=6.71 KB/6870 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 1249ms, sequenceid=524, compaction requested=false 2024-11-28T03:43:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2603): Flush status journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: 2024-11-28T03:43:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154 2024-11-28T03:43:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=154 2024-11-28T03:43:18,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-28T03:43:18,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7110 sec 2024-11-28T03:43:18,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees in 1.7150 sec 2024-11-28T03:43:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-28T03:43:18,617 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:21,698 INFO [Thread-2201 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2201. 2024-11-28T03:43:21,698 DEBUG [Thread-2201 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:21,698 DEBUG [Thread-2201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:21,698 DEBUG [Thread-2201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:21,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:21,738 INFO [Thread-2207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2207. 2024-11-28T03:43:21,738 DEBUG [Thread-2207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:21,738 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:21,738 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:21,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:21,798 INFO [Thread-2199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2199. 2024-11-28T03:43:21,798 DEBUG [Thread-2199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:21,798 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:21,798 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:21,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:21,807 INFO [Thread-2205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2205. 2024-11-28T03:43:21,807 DEBUG [Thread-2205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:21,807 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:21,807 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:21,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 288 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3303 rows 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1106 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3318 rows 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1099 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3297 rows 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1101 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3303 rows 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1108 2024-11-28T03:43:21,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3322 rows 2024-11-28T03:43:21,808 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:43:21,808 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:43:21,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:21,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:21,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:21,809 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T03:43:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-28T03:43:21,811 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765401811"}]},"ts":"1732765401811"} 2024-11-28T03:43:21,812 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T03:43:21,813 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T03:43:21,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T03:43:21,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2c8ad458d2ec9e8bad5d996a8a6cef0, UNASSIGN}] 2024-11-28T03:43:21,815 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2c8ad458d2ec9e8bad5d996a8a6cef0, UNASSIGN 2024-11-28T03:43:21,815 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=e2c8ad458d2ec9e8bad5d996a8a6cef0, regionState=CLOSING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:43:21,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2c8ad458d2ec9e8bad5d996a8a6cef0, UNASSIGN because future has completed 2024-11-28T03:43:21,817 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T03:43:21,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:43:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-28T03:43:21,969 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing e2c8ad458d2ec9e8bad5d996a8a6cef0, disabling compactions & flushes 2024-11-28T03:43:21,969 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. after waiting 0 ms 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:21,969 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(2902): Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=A 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=B 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e2c8ad458d2ec9e8bad5d996a8a6cef0, store=C 2024-11-28T03:43:21,969 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:21,973 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/0983eebe906342f89f12364bb46ef3f4 is 50, key is test_row_0/A:col10/1732765401797/Put/seqid=0 2024-11-28T03:43:21,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742437_1613 (size=12301) 2024-11-28T03:43:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-28T03:43:22,376 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/0983eebe906342f89f12364bb46ef3f4 2024-11-28T03:43:22,382 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/8c357723420f4b328a791fb8e55c396e is 50, key is test_row_0/B:col10/1732765401797/Put/seqid=0 2024-11-28T03:43:22,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742438_1614 (size=12301) 2024-11-28T03:43:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-28T03:43:22,786 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/8c357723420f4b328a791fb8e55c396e 2024-11-28T03:43:22,791 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/c773d735d22446f7a2717b23dbafea05 is 50, key is test_row_0/C:col10/1732765401797/Put/seqid=0 2024-11-28T03:43:22,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742439_1615 (size=12301) 2024-11-28T03:43:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-28T03:43:23,195 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/c773d735d22446f7a2717b23dbafea05 2024-11-28T03:43:23,199 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/A/0983eebe906342f89f12364bb46ef3f4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/0983eebe906342f89f12364bb46ef3f4 2024-11-28T03:43:23,202 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/0983eebe906342f89f12364bb46ef3f4, entries=150, sequenceid=535, filesize=12.0 K 2024-11-28T03:43:23,202 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/B/8c357723420f4b328a791fb8e55c396e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/8c357723420f4b328a791fb8e55c396e 2024-11-28T03:43:23,205 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/8c357723420f4b328a791fb8e55c396e, entries=150, sequenceid=535, filesize=12.0 K 2024-11-28T03:43:23,206 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/.tmp/C/c773d735d22446f7a2717b23dbafea05 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/c773d735d22446f7a2717b23dbafea05 2024-11-28T03:43:23,209 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/c773d735d22446f7a2717b23dbafea05, entries=150, sequenceid=535, filesize=12.0 K 2024-11-28T03:43:23,210 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 1241ms, sequenceid=535, compaction requested=true 2024-11-28T03:43:23,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d50a5e94259465481448a4fdd34b6df, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c45246a53ef7450b9f5d5d1161acfbda, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3b0b28f02843434e875a01e569d4b650, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2b0b3752e4df4b04ba0ce94abbc85165, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/62ea1ca62e684ff6ae4ee323d5cce224, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/190752297ba145658b409c8ffcab6772, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/69337a3939bf49e2a31dbad74f8762dd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/984e878f22a041449c8a9a00d55d561d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7f1b8ff923d44ee2ae5c1e97f6de0783, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c07e7bd602b64d59a878ebe063d39f50, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d41201541fe944b2a56d23ec850c6e62, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/257465698dcd4f36b6eb0650953cad10, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2de80b4e4a694761bfd9826c76a8fc89, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2ad98151756c4019bb30c42087a86647, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7407a931bd31492ba203647bb2901565, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d40f237156046e8ba942c972858fbb1, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/1f20ca871f254acba9e94e5ea6372ba5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3193b8fcecdf4132baa840b99c2b0971, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52e33a414c364985985b72a11037de42, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/addf0d8df24a4800ae080dbb53411149, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/61c26fb5bfec400db1c370ef9c573b0d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/e6183ea84f3c40c78a12c78e735fe428, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/1d4cda1bf2884b45a61e935c0789152f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/469e5992d72b4f5b8f96254e28ab1ebe, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/cb0502b1274848f48dcf93d27461664e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/76b835ffac614296a4c5f848f9cb5570, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/fd016975d6194f0587a3ef6f79d14251, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/526bc973c7944151a22e318b0036c711, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52b1edc7989442c8b4b9169fd5258f74, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d5912243e08243f0a10f45e8081d0852, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7022fd289ab243d4807bffe7f7e286e9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2bc596e08e8149d7892169043cbece33, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d0ceeef721db423ea16e15e53eb86f13, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9436b825208347988d9f9ef0c03d2d5c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/f482d86613a94dd4a1e7b1a0bb28c4d6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/60be5800dbf14c4eab7e87190cf19e5c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/45b2fefa53f944aa9e6a1a4cb1e06243] to archive 2024-11-28T03:43:23,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:43:23,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d50a5e94259465481448a4fdd34b6df to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d50a5e94259465481448a4fdd34b6df 2024-11-28T03:43:23,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c45246a53ef7450b9f5d5d1161acfbda to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c45246a53ef7450b9f5d5d1161acfbda 2024-11-28T03:43:23,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3b0b28f02843434e875a01e569d4b650 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3b0b28f02843434e875a01e569d4b650 2024-11-28T03:43:23,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2b0b3752e4df4b04ba0ce94abbc85165 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2b0b3752e4df4b04ba0ce94abbc85165 2024-11-28T03:43:23,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/62ea1ca62e684ff6ae4ee323d5cce224 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/62ea1ca62e684ff6ae4ee323d5cce224 2024-11-28T03:43:23,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/190752297ba145658b409c8ffcab6772 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/190752297ba145658b409c8ffcab6772 2024-11-28T03:43:23,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/69337a3939bf49e2a31dbad74f8762dd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/69337a3939bf49e2a31dbad74f8762dd 2024-11-28T03:43:23,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/984e878f22a041449c8a9a00d55d561d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/984e878f22a041449c8a9a00d55d561d 2024-11-28T03:43:23,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7f1b8ff923d44ee2ae5c1e97f6de0783 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7f1b8ff923d44ee2ae5c1e97f6de0783 2024-11-28T03:43:23,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c07e7bd602b64d59a878ebe063d39f50 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c07e7bd602b64d59a878ebe063d39f50 2024-11-28T03:43:23,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d41201541fe944b2a56d23ec850c6e62 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d41201541fe944b2a56d23ec850c6e62 2024-11-28T03:43:23,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/257465698dcd4f36b6eb0650953cad10 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/257465698dcd4f36b6eb0650953cad10 2024-11-28T03:43:23,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2de80b4e4a694761bfd9826c76a8fc89 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2de80b4e4a694761bfd9826c76a8fc89 2024-11-28T03:43:23,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2ad98151756c4019bb30c42087a86647 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2ad98151756c4019bb30c42087a86647 2024-11-28T03:43:23,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7407a931bd31492ba203647bb2901565 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7407a931bd31492ba203647bb2901565 2024-11-28T03:43:23,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d40f237156046e8ba942c972858fbb1 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9d40f237156046e8ba942c972858fbb1 2024-11-28T03:43:23,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/1f20ca871f254acba9e94e5ea6372ba5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/1f20ca871f254acba9e94e5ea6372ba5 2024-11-28T03:43:23,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3193b8fcecdf4132baa840b99c2b0971 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/3193b8fcecdf4132baa840b99c2b0971 2024-11-28T03:43:23,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52e33a414c364985985b72a11037de42 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52e33a414c364985985b72a11037de42 2024-11-28T03:43:23,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/addf0d8df24a4800ae080dbb53411149 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/addf0d8df24a4800ae080dbb53411149 2024-11-28T03:43:23,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/61c26fb5bfec400db1c370ef9c573b0d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/61c26fb5bfec400db1c370ef9c573b0d 2024-11-28T03:43:23,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/e6183ea84f3c40c78a12c78e735fe428 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/e6183ea84f3c40c78a12c78e735fe428 2024-11-28T03:43:23,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/1d4cda1bf2884b45a61e935c0789152f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/1d4cda1bf2884b45a61e935c0789152f 2024-11-28T03:43:23,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/469e5992d72b4f5b8f96254e28ab1ebe to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/469e5992d72b4f5b8f96254e28ab1ebe 2024-11-28T03:43:23,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/cb0502b1274848f48dcf93d27461664e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/cb0502b1274848f48dcf93d27461664e 2024-11-28T03:43:23,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/76b835ffac614296a4c5f848f9cb5570 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/76b835ffac614296a4c5f848f9cb5570 2024-11-28T03:43:23,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/fd016975d6194f0587a3ef6f79d14251 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/fd016975d6194f0587a3ef6f79d14251 2024-11-28T03:43:23,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/526bc973c7944151a22e318b0036c711 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/526bc973c7944151a22e318b0036c711 2024-11-28T03:43:23,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52b1edc7989442c8b4b9169fd5258f74 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/52b1edc7989442c8b4b9169fd5258f74 2024-11-28T03:43:23,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d5912243e08243f0a10f45e8081d0852 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d5912243e08243f0a10f45e8081d0852 2024-11-28T03:43:23,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7022fd289ab243d4807bffe7f7e286e9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/7022fd289ab243d4807bffe7f7e286e9 2024-11-28T03:43:23,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2bc596e08e8149d7892169043cbece33 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/2bc596e08e8149d7892169043cbece33 2024-11-28T03:43:23,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d0ceeef721db423ea16e15e53eb86f13 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/d0ceeef721db423ea16e15e53eb86f13 2024-11-28T03:43:23,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9436b825208347988d9f9ef0c03d2d5c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/9436b825208347988d9f9ef0c03d2d5c 2024-11-28T03:43:23,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/f482d86613a94dd4a1e7b1a0bb28c4d6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/f482d86613a94dd4a1e7b1a0bb28c4d6 2024-11-28T03:43:23,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/60be5800dbf14c4eab7e87190cf19e5c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/60be5800dbf14c4eab7e87190cf19e5c 2024-11-28T03:43:23,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/45b2fefa53f944aa9e6a1a4cb1e06243 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/45b2fefa53f944aa9e6a1a4cb1e06243 2024-11-28T03:43:23,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e7d14e54cfb04ffd8dba9f747f41ea98, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e79965e9e84142f1bbef24753fc65c55, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/f7bb804e42f3493d87b07ea05d34cc52, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d92c53ca08ff4062a70794b6bac84506, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0f130531ca39472faec932d42211d4ad, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0a55b5d874e5486a8c38c5fc6c02010d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/66741cc480e742be99fa9ac29fbde53f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/9453e353dc2043738e4130f90f9388bf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/2d0210f662df4b90ab3ef55655e45ee9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/7c13ff1eb46345c281398d94ca86c685, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/26862949bff7458785b334bef89b447b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/93cca20a874344158c74c4601910b23a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/645ded5a2cc04a04854df20099600a38, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cee0f1921c5241bd9b434ca87f928569, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ed3132dbc1a1470fbe2825aa66fa444e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/46e1a0fdd95e40b7a402991df34e4dd5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4f0c494e09044fc4937f2ff42e6b4ca3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cf90fbc4bc0945bf8b31ce82d05f2faf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/32beaabbf4e34226a7348afaf3c74a9d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/eb7bd3560ea14988b6c53bceb8d6473c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c75c7c78e7c74b619719afee20aa7977, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6350db788e4548de8b831571b54a065c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c77dafac77064a03aa36f90727929cac, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/496351a491da4c3cbba2f9f9a06058b7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/734429a3c084428baec1f35daa84c9d4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b0f239e40338408aa0fd777f5f338b98, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/23b2cc0070ea4830a448c87906d3de3f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/02a12ca28d3b4264985dc571bc298ac9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6aa8ae3f1cdd465d83107de816359336, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/5967c1a6beea4b61a689bfc653a73eae, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/37dbb990318143b18109efb7fe81d311, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/74b889f1e4d74506a5978fb34a562196, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d408621c06be4444869b1d00ceebe111, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4309fd119168443d84a04f2a3fef1e58, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ea103568c43a4644b360a954e7ac3e25, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b1b0e01d4ba147f1ba8df40b06726411, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/80e39915d04943798865dc00b86d481f] to archive 2024-11-28T03:43:23,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:43:23,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e7d14e54cfb04ffd8dba9f747f41ea98 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e7d14e54cfb04ffd8dba9f747f41ea98 2024-11-28T03:43:23,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e79965e9e84142f1bbef24753fc65c55 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/e79965e9e84142f1bbef24753fc65c55 2024-11-28T03:43:23,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/f7bb804e42f3493d87b07ea05d34cc52 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/f7bb804e42f3493d87b07ea05d34cc52 2024-11-28T03:43:23,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d92c53ca08ff4062a70794b6bac84506 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d92c53ca08ff4062a70794b6bac84506 2024-11-28T03:43:23,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0f130531ca39472faec932d42211d4ad to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0f130531ca39472faec932d42211d4ad 2024-11-28T03:43:23,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0a55b5d874e5486a8c38c5fc6c02010d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/0a55b5d874e5486a8c38c5fc6c02010d 2024-11-28T03:43:23,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/66741cc480e742be99fa9ac29fbde53f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/66741cc480e742be99fa9ac29fbde53f 2024-11-28T03:43:23,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/9453e353dc2043738e4130f90f9388bf to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/9453e353dc2043738e4130f90f9388bf 2024-11-28T03:43:23,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/2d0210f662df4b90ab3ef55655e45ee9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/2d0210f662df4b90ab3ef55655e45ee9 2024-11-28T03:43:23,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/7c13ff1eb46345c281398d94ca86c685 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/7c13ff1eb46345c281398d94ca86c685 2024-11-28T03:43:23,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/26862949bff7458785b334bef89b447b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/26862949bff7458785b334bef89b447b 2024-11-28T03:43:23,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/93cca20a874344158c74c4601910b23a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/93cca20a874344158c74c4601910b23a 2024-11-28T03:43:23,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/645ded5a2cc04a04854df20099600a38 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/645ded5a2cc04a04854df20099600a38 2024-11-28T03:43:23,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cee0f1921c5241bd9b434ca87f928569 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cee0f1921c5241bd9b434ca87f928569 2024-11-28T03:43:23,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ed3132dbc1a1470fbe2825aa66fa444e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ed3132dbc1a1470fbe2825aa66fa444e 2024-11-28T03:43:23,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/46e1a0fdd95e40b7a402991df34e4dd5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/46e1a0fdd95e40b7a402991df34e4dd5 2024-11-28T03:43:23,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4f0c494e09044fc4937f2ff42e6b4ca3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4f0c494e09044fc4937f2ff42e6b4ca3 2024-11-28T03:43:23,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cf90fbc4bc0945bf8b31ce82d05f2faf to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/cf90fbc4bc0945bf8b31ce82d05f2faf 2024-11-28T03:43:23,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/32beaabbf4e34226a7348afaf3c74a9d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/32beaabbf4e34226a7348afaf3c74a9d 2024-11-28T03:43:23,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/eb7bd3560ea14988b6c53bceb8d6473c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/eb7bd3560ea14988b6c53bceb8d6473c 2024-11-28T03:43:23,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c75c7c78e7c74b619719afee20aa7977 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c75c7c78e7c74b619719afee20aa7977 2024-11-28T03:43:23,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6350db788e4548de8b831571b54a065c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6350db788e4548de8b831571b54a065c 2024-11-28T03:43:23,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c77dafac77064a03aa36f90727929cac to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/c77dafac77064a03aa36f90727929cac 2024-11-28T03:43:23,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/496351a491da4c3cbba2f9f9a06058b7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/496351a491da4c3cbba2f9f9a06058b7 2024-11-28T03:43:23,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/734429a3c084428baec1f35daa84c9d4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/734429a3c084428baec1f35daa84c9d4 2024-11-28T03:43:23,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b0f239e40338408aa0fd777f5f338b98 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b0f239e40338408aa0fd777f5f338b98 2024-11-28T03:43:23,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/23b2cc0070ea4830a448c87906d3de3f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/23b2cc0070ea4830a448c87906d3de3f 2024-11-28T03:43:23,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/02a12ca28d3b4264985dc571bc298ac9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/02a12ca28d3b4264985dc571bc298ac9 2024-11-28T03:43:23,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6aa8ae3f1cdd465d83107de816359336 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/6aa8ae3f1cdd465d83107de816359336 2024-11-28T03:43:23,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/5967c1a6beea4b61a689bfc653a73eae to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/5967c1a6beea4b61a689bfc653a73eae 2024-11-28T03:43:23,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/37dbb990318143b18109efb7fe81d311 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/37dbb990318143b18109efb7fe81d311 2024-11-28T03:43:23,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/74b889f1e4d74506a5978fb34a562196 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/74b889f1e4d74506a5978fb34a562196 2024-11-28T03:43:23,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d408621c06be4444869b1d00ceebe111 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/d408621c06be4444869b1d00ceebe111 2024-11-28T03:43:23,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4309fd119168443d84a04f2a3fef1e58 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/4309fd119168443d84a04f2a3fef1e58 2024-11-28T03:43:23,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ea103568c43a4644b360a954e7ac3e25 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/ea103568c43a4644b360a954e7ac3e25 2024-11-28T03:43:23,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b1b0e01d4ba147f1ba8df40b06726411 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/b1b0e01d4ba147f1ba8df40b06726411 2024-11-28T03:43:23,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/80e39915d04943798865dc00b86d481f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/80e39915d04943798865dc00b86d481f 2024-11-28T03:43:23,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5143253c79574eb586605aaf2a9a433b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/082357dfcf5f462ea78df07b69e52fa7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/7423efab26cb413bab1d8d086b75aff9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6a514c9e907d4373b9c49de5a1ff36de, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e59dbf02e97e448c9383b2728f682b4c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/487508693b164942a49cc5fb62201524, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e63a2554cfbc48919b6d10e22586a0f4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e2b21aade3fc4411ba31c779a2797c68, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/a80fbbb17ca04dbb8f07276f4a2ce8cf, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5fab8497b427420c9d5ddd2bd4eebcb6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5c30d6b02d914ffb8d82774240c9d93a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/aeaf8709dba4448899cd3eb244d42a30, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c8e811f1cd24f5fb18dced88e8103ef, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5ca1089a6ecf4564b095bdafc7cbf3c2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2b2d639cdc6940d0aef306be4af05eff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/d0a983c851d74d7091e57a653d8f9d48, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5bc4bf80f8374e32b0de829452e066c4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/bf4d93c104404f3284f6f0d999072861, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6e96c8d855bb4351ad28134fdc98b01c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4faa3267b3424161b213252cdb39d3c6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4d20d052eba94cf3a8504b0dd5f22b4c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/b78b3e0a6277447dbe77960ea9798143, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2d845ab070724c7da9338609ad77c785, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a5655f8af9543b98709df91f9485f9d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/c01246644cc84172827448b96e69f54c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8d8c23357d2046b78aaefe694984ea80, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/9f0444f2f8a445a7b338d40a306317bd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/46e31fad10394d908c971e9d07e3b1c7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/0da186b58d5745c6907e829b2c464cb4, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/42b033fdf9c0401c8329471392f5a18f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/ea4d9a753c174bd4b735397751a6937d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c3f29ebc2af40fab4fcb7c607edf144, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/1b8cc1e8952c4196936d9f9fa18a19d2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/155bc0d4ecd44697b6af7d36f53d87f6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a16968c88524089b37c09732344355f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/df7da848638f466dad271e77d9710d96, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/06c52439e7f344a9b1c491a6dfb34188] to archive 2024-11-28T03:43:23,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:43:23,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5143253c79574eb586605aaf2a9a433b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5143253c79574eb586605aaf2a9a433b 2024-11-28T03:43:23,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/082357dfcf5f462ea78df07b69e52fa7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/082357dfcf5f462ea78df07b69e52fa7 2024-11-28T03:43:23,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/7423efab26cb413bab1d8d086b75aff9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/7423efab26cb413bab1d8d086b75aff9 2024-11-28T03:43:23,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6a514c9e907d4373b9c49de5a1ff36de to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6a514c9e907d4373b9c49de5a1ff36de 2024-11-28T03:43:23,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e59dbf02e97e448c9383b2728f682b4c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e59dbf02e97e448c9383b2728f682b4c 2024-11-28T03:43:23,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/487508693b164942a49cc5fb62201524 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/487508693b164942a49cc5fb62201524 2024-11-28T03:43:23,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e63a2554cfbc48919b6d10e22586a0f4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e63a2554cfbc48919b6d10e22586a0f4 2024-11-28T03:43:23,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e2b21aade3fc4411ba31c779a2797c68 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/e2b21aade3fc4411ba31c779a2797c68 2024-11-28T03:43:23,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/a80fbbb17ca04dbb8f07276f4a2ce8cf to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/a80fbbb17ca04dbb8f07276f4a2ce8cf 2024-11-28T03:43:23,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5fab8497b427420c9d5ddd2bd4eebcb6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5fab8497b427420c9d5ddd2bd4eebcb6 2024-11-28T03:43:23,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5c30d6b02d914ffb8d82774240c9d93a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5c30d6b02d914ffb8d82774240c9d93a 2024-11-28T03:43:23,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/aeaf8709dba4448899cd3eb244d42a30 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/aeaf8709dba4448899cd3eb244d42a30 2024-11-28T03:43:23,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c8e811f1cd24f5fb18dced88e8103ef to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c8e811f1cd24f5fb18dced88e8103ef 2024-11-28T03:43:23,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5ca1089a6ecf4564b095bdafc7cbf3c2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5ca1089a6ecf4564b095bdafc7cbf3c2 2024-11-28T03:43:23,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2b2d639cdc6940d0aef306be4af05eff to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2b2d639cdc6940d0aef306be4af05eff 2024-11-28T03:43:23,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/d0a983c851d74d7091e57a653d8f9d48 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/d0a983c851d74d7091e57a653d8f9d48 2024-11-28T03:43:23,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5bc4bf80f8374e32b0de829452e066c4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5bc4bf80f8374e32b0de829452e066c4 2024-11-28T03:43:23,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/bf4d93c104404f3284f6f0d999072861 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/bf4d93c104404f3284f6f0d999072861 2024-11-28T03:43:23,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6e96c8d855bb4351ad28134fdc98b01c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/6e96c8d855bb4351ad28134fdc98b01c 2024-11-28T03:43:23,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4faa3267b3424161b213252cdb39d3c6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4faa3267b3424161b213252cdb39d3c6 2024-11-28T03:43:23,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4d20d052eba94cf3a8504b0dd5f22b4c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/4d20d052eba94cf3a8504b0dd5f22b4c 2024-11-28T03:43:23,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/b78b3e0a6277447dbe77960ea9798143 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/b78b3e0a6277447dbe77960ea9798143 2024-11-28T03:43:23,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2d845ab070724c7da9338609ad77c785 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/2d845ab070724c7da9338609ad77c785 2024-11-28T03:43:23,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a5655f8af9543b98709df91f9485f9d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a5655f8af9543b98709df91f9485f9d 2024-11-28T03:43:23,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/c01246644cc84172827448b96e69f54c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/c01246644cc84172827448b96e69f54c 2024-11-28T03:43:23,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8d8c23357d2046b78aaefe694984ea80 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8d8c23357d2046b78aaefe694984ea80 2024-11-28T03:43:23,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/9f0444f2f8a445a7b338d40a306317bd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/9f0444f2f8a445a7b338d40a306317bd 2024-11-28T03:43:23,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/46e31fad10394d908c971e9d07e3b1c7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/46e31fad10394d908c971e9d07e3b1c7 2024-11-28T03:43:23,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/0da186b58d5745c6907e829b2c464cb4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/0da186b58d5745c6907e829b2c464cb4 2024-11-28T03:43:23,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/42b033fdf9c0401c8329471392f5a18f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/42b033fdf9c0401c8329471392f5a18f 2024-11-28T03:43:23,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/ea4d9a753c174bd4b735397751a6937d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/ea4d9a753c174bd4b735397751a6937d 2024-11-28T03:43:23,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c3f29ebc2af40fab4fcb7c607edf144 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8c3f29ebc2af40fab4fcb7c607edf144 2024-11-28T03:43:23,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/1b8cc1e8952c4196936d9f9fa18a19d2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/1b8cc1e8952c4196936d9f9fa18a19d2 2024-11-28T03:43:23,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/155bc0d4ecd44697b6af7d36f53d87f6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/155bc0d4ecd44697b6af7d36f53d87f6 2024-11-28T03:43:23,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a16968c88524089b37c09732344355f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/5a16968c88524089b37c09732344355f 2024-11-28T03:43:23,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/df7da848638f466dad271e77d9710d96 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/df7da848638f466dad271e77d9710d96 2024-11-28T03:43:23,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/06c52439e7f344a9b1c491a6dfb34188 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/06c52439e7f344a9b1c491a6dfb34188 2024-11-28T03:43:23,313 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/recovered.edits/538.seqid, newMaxSeqId=538, maxSeqId=1 2024-11-28T03:43:23,314 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. 2024-11-28T03:43:23,314 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for e2c8ad458d2ec9e8bad5d996a8a6cef0: Waiting for close lock at 1732765401969Running coprocessor pre-close hooks at 1732765401969Disabling compacts and flushes for region at 1732765401969Disabling writes for close at 1732765401969Obtaining lock to block concurrent updates at 1732765401969Preparing flush snapshotting stores in e2c8ad458d2ec9e8bad5d996a8a6cef0 at 1732765401969Finished memstore snapshotting TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732765401969Flushing stores of TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0. at 1732765401970 (+1 ms)Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0/A: creating writer at 1732765401970Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0/A: appending metadata at 1732765401972 (+2 ms)Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0/A: closing flushed file at 1732765401972Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0/B: creating writer at 1732765402380 (+408 ms)Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0/B: appending metadata at 1732765402382 (+2 ms)Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0/B: closing flushed file at 1732765402382Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0/C: creating writer at 1732765402789 (+407 ms)Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0/C: appending metadata at 1732765402791 (+2 ms)Flushing e2c8ad458d2ec9e8bad5d996a8a6cef0/C: closing flushed file at 1732765402791Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e96b558: reopening flushed file at 1732765403198 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cf66c80: reopening flushed file at 1732765403202 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e1a5859: reopening flushed file at 1732765403205 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e2c8ad458d2ec9e8bad5d996a8a6cef0 in 1241ms, sequenceid=535, compaction requested=true at 1732765403210 (+5 ms)Writing region close event to WAL at 1732765403311 (+101 ms)Running coprocessor post-close hooks at 1732765403314 (+3 ms)Closed at 1732765403314 2024-11-28T03:43:23,316 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:23,316 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=e2c8ad458d2ec9e8bad5d996a8a6cef0, regionState=CLOSED 2024-11-28T03:43:23,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:43:23,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-28T03:43:23,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure e2c8ad458d2ec9e8bad5d996a8a6cef0, server=3a546af21488,34671,1732765249447 in 1.5010 sec 2024-11-28T03:43:23,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-28T03:43:23,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2c8ad458d2ec9e8bad5d996a8a6cef0, UNASSIGN in 1.5050 sec 2024-11-28T03:43:23,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-28T03:43:23,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5080 sec 2024-11-28T03:43:23,324 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765403324"}]},"ts":"1732765403324"} 2024-11-28T03:43:23,325 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T03:43:23,325 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T03:43:23,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5160 sec 2024-11-28T03:43:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-28T03:43:23,957 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:23,957 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T03:43:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=159, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:23,959 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=159, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-11-28T03:43:23,959 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=159, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:23,961 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:23,963 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/recovered.edits] 2024-11-28T03:43:23,965 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/0983eebe906342f89f12364bb46ef3f4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/0983eebe906342f89f12364bb46ef3f4 2024-11-28T03:43:23,966 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c1bcc3f05d8241e8bee5e707dad576b6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/c1bcc3f05d8241e8bee5e707dad576b6 2024-11-28T03:43:23,966 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/e0bfb6e853534c49a35e9e99c621b5fe to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/A/e0bfb6e853534c49a35e9e99c621b5fe 2024-11-28T03:43:23,968 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/1ab965d3b32245a7a023ce4cfd39c790 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/1ab965d3b32245a7a023ce4cfd39c790 2024-11-28T03:43:23,969 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/31bccf942fcb45e0aa0f77859dd33bc4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/31bccf942fcb45e0aa0f77859dd33bc4 2024-11-28T03:43:23,969 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/8c357723420f4b328a791fb8e55c396e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/B/8c357723420f4b328a791fb8e55c396e 2024-11-28T03:43:23,971 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/270b698a7b60446f94b4440672618c5d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/270b698a7b60446f94b4440672618c5d 2024-11-28T03:43:23,972 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8d4916840e7b4564b24c777bbaaae8bb to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/8d4916840e7b4564b24c777bbaaae8bb 2024-11-28T03:43:23,972 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/c773d735d22446f7a2717b23dbafea05 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/C/c773d735d22446f7a2717b23dbafea05 2024-11-28T03:43:23,974 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/recovered.edits/538.seqid to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0/recovered.edits/538.seqid 2024-11-28T03:43:23,975 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/e2c8ad458d2ec9e8bad5d996a8a6cef0 2024-11-28T03:43:23,975 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T03:43:23,976 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=159, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:23,979 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T03:43:23,980 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T03:43:23,981 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=159, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:23,981 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T03:43:23,982 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732765403981"}]},"ts":"9223372036854775807"} 2024-11-28T03:43:23,983 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T03:43:23,983 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e2c8ad458d2ec9e8bad5d996a8a6cef0, NAME => 'TestAcidGuarantees,,1732765374527.e2c8ad458d2ec9e8bad5d996a8a6cef0.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T03:43:23,983 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T03:43:23,983 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732765403983"}]},"ts":"9223372036854775807"} 2024-11-28T03:43:23,984 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T03:43:23,985 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=159, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:23,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-28T03:43:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-11-28T03:43:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T03:43:24,067 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:24,076 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 252), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=493 (was 478) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6044 (was 6183) 2024-11-28T03:43:24,084 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=493, ProcessCount=11, AvailableMemoryMB=6044 2024-11-28T03:43:24,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T03:43:24,086 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:43:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:24,087 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=160, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:43:24,087 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:24,087 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 160 2024-11-28T03:43:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T03:43:24,088 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=160, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:43:24,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742440_1616 (size=1061) 2024-11-28T03:43:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T03:43:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T03:43:24,494 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72 2024-11-28T03:43:24,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742441_1617 (size=53) 2024-11-28T03:43:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T03:43:24,899 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:43:24,900 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6c2355f8fea173e178470cf1c753f473, disabling compactions & flushes 2024-11-28T03:43:24,900 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:24,900 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:24,900 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. after waiting 0 ms 2024-11-28T03:43:24,900 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:24,900 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:24,900 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6c2355f8fea173e178470cf1c753f473: Waiting for close lock at 1732765404899Disabling compacts and flushes for region at 1732765404900 (+1 ms)Disabling writes for close at 1732765404900Writing region close event to WAL at 1732765404900Closed at 1732765404900 2024-11-28T03:43:24,901 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=160, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:43:24,901 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732765404901"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732765404901"}]},"ts":"1732765404901"} 2024-11-28T03:43:24,902 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:43:24,903 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=160, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:43:24,903 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765404903"}]},"ts":"1732765404903"} 2024-11-28T03:43:24,904 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T03:43:24,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, ASSIGN}] 2024-11-28T03:43:24,905 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, ASSIGN 2024-11-28T03:43:24,906 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, ASSIGN; state=OFFLINE, location=3a546af21488,34671,1732765249447; forceNewPlan=false, retain=false 2024-11-28T03:43:25,056 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=161 updating hbase:meta row=6c2355f8fea173e178470cf1c753f473, regionState=OPENING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:43:25,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, ASSIGN because future has completed 2024-11-28T03:43:25,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:43:25,213 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:25,213 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:43:25,213 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,213 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:43:25,214 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(7794): checking encryption for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,214 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(7797): checking classloading for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,215 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,216 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:43:25,216 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c2355f8fea173e178470cf1c753f473 columnFamilyName A 2024-11-28T03:43:25,216 DEBUG [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:25,217 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(327): Store=6c2355f8fea173e178470cf1c753f473/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:43:25,217 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,218 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:43:25,218 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c2355f8fea173e178470cf1c753f473 columnFamilyName B 2024-11-28T03:43:25,218 DEBUG [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:25,218 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(327): Store=6c2355f8fea173e178470cf1c753f473/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:43:25,218 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,219 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:43:25,220 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c2355f8fea173e178470cf1c753f473 columnFamilyName C 2024-11-28T03:43:25,220 DEBUG [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:25,220 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(327): Store=6c2355f8fea173e178470cf1c753f473/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:43:25,220 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:25,220 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(1038): replaying wal for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,221 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,221 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,222 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(1048): stopping wal replay for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,222 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(1060): Cleaning up temporary data for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,223 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:43:25,224 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(1093): writing seq id for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,226 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:43:25,226 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(1114): Opened 6c2355f8fea173e178470cf1c753f473; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60379533, jitterRate=-0.10027484595775604}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:43:25,226 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T03:43:25,227 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegion(1006): Region open journal for 6c2355f8fea173e178470cf1c753f473: Running coprocessor pre-open hook at 1732765405214Writing region info on filesystem at 1732765405214Initializing all the Stores at 1732765405214Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765405214Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765405215 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765405215Cleaning up temporary data from old regions at 1732765405222 (+7 ms)Running coprocessor post-open hooks at 1732765405226 (+4 ms)Region opened successfully at 1732765405226 2024-11-28T03:43:25,227 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., pid=162, masterSystemTime=1732765405210 2024-11-28T03:43:25,229 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:25,229 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=162}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:25,230 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=161 updating hbase:meta row=6c2355f8fea173e178470cf1c753f473, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:43:25,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:43:25,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-28T03:43:25,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 in 174 msec 2024-11-28T03:43:25,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-28T03:43:25,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, ASSIGN in 330 msec 2024-11-28T03:43:25,236 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=160, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:43:25,236 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765405236"}]},"ts":"1732765405236"} 2024-11-28T03:43:25,237 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T03:43:25,238 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=160, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:43:25,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-28T03:43:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T03:43:26,236 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:26,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138e8802, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:26,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:26,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:26,238 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da31b00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:26,240 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b7d8dc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:26,241 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:26,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:26,243 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T03:43:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 3a546af21488,38327,1732765248628 2024-11-28T03:43:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@97a9c34 2024-11-28T03:43:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T03:43:26,245 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T03:43:26,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T03:43:26,246 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:43:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=163, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:26,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742442_1618 (size=1097) 2024-11-28T03:43:26,656 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T03:43:26,656 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T03:43:26,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T03:43:26,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, REOPEN/MOVE}] 2024-11-28T03:43:26,660 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, REOPEN/MOVE 2024-11-28T03:43:26,661 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=165 updating hbase:meta row=6c2355f8fea173e178470cf1c753f473, regionState=CLOSING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:43:26,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, REOPEN/MOVE because future has completed 2024-11-28T03:43:26,663 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T03:43:26,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:43:26,815 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(122): Close 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:26,815 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T03:43:26,815 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1722): Closing 6c2355f8fea173e178470cf1c753f473, disabling compactions & flushes 2024-11-28T03:43:26,815 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:26,815 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:26,815 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. after waiting 0 ms 2024-11-28T03:43:26,815 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:26,818 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T03:43:26,819 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:26,819 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1676): Region close journal for 6c2355f8fea173e178470cf1c753f473: Waiting for close lock at 1732765406815Running coprocessor pre-close hooks at 1732765406815Disabling compacts and flushes for region at 1732765406815Disabling writes for close at 1732765406815Writing region close event to WAL at 1732765406815Running coprocessor post-close hooks at 1732765406819 (+4 ms)Closed at 1732765406819 2024-11-28T03:43:26,819 WARN [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegionServer(3299): Not adding moved region record: 6c2355f8fea173e178470cf1c753f473 to self. 2024-11-28T03:43:26,820 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(157): Closed 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:26,821 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=165 updating hbase:meta row=6c2355f8fea173e178470cf1c753f473, regionState=CLOSED 2024-11-28T03:43:26,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:43:26,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-28T03:43:26,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; CloseRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 in 159 msec 2024-11-28T03:43:26,824 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, REOPEN/MOVE; state=CLOSED, location=3a546af21488,34671,1732765249447; forceNewPlan=false, retain=true 2024-11-28T03:43:26,975 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=165 updating hbase:meta row=6c2355f8fea173e178470cf1c753f473, regionState=OPENING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:43:26,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, REOPEN/MOVE because future has completed 2024-11-28T03:43:26,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=165, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:43:27,130 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,130 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:43:27,131 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,131 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:43:27,131 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(7794): checking encryption for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,131 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(7797): checking classloading for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,132 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,133 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:43:27,133 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c2355f8fea173e178470cf1c753f473 columnFamilyName A 2024-11-28T03:43:27,134 DEBUG [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:27,134 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(327): Store=6c2355f8fea173e178470cf1c753f473/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:43:27,135 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,135 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:43:27,135 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c2355f8fea173e178470cf1c753f473 columnFamilyName B 2024-11-28T03:43:27,135 DEBUG [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:27,136 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(327): Store=6c2355f8fea173e178470cf1c753f473/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:43:27,136 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,136 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T03:43:27,136 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c2355f8fea173e178470cf1c753f473 columnFamilyName C 2024-11-28T03:43:27,136 DEBUG [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:27,137 INFO [StoreOpener-6c2355f8fea173e178470cf1c753f473-1 {}] regionserver.HStore(327): Store=6c2355f8fea173e178470cf1c753f473/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:43:27,137 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,137 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(1038): replaying wal for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,137 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,138 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,139 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(1048): stopping wal replay for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,139 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(1060): Cleaning up temporary data for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,139 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T03:43:27,140 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(1093): writing seq id for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,141 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(1114): Opened 6c2355f8fea173e178470cf1c753f473; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68624216, jitterRate=0.022580504417419434}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T03:43:27,141 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,141 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegion(1006): Region open journal for 6c2355f8fea173e178470cf1c753f473: Running coprocessor pre-open hook at 1732765407131Writing region info on filesystem at 1732765407131Initializing all the Stores at 1732765407132 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765407132Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765407132Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732765407132Cleaning up temporary data from old regions at 1732765407139 (+7 ms)Running coprocessor post-open hooks at 1732765407141 (+2 ms)Region opened successfully at 1732765407141 2024-11-28T03:43:27,142 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., pid=167, masterSystemTime=1732765407128 2024-11-28T03:43:27,143 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,143 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=167}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,144 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=165 updating hbase:meta row=6c2355f8fea173e178470cf1c753f473, regionState=OPEN, openSeqNum=5, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=165, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:43:27,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=165 2024-11-28T03:43:27,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=165, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 in 170 msec 2024-11-28T03:43:27,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-28T03:43:27,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, REOPEN/MOVE in 487 msec 2024-11-28T03:43:27,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-28T03:43:27,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-28T03:43:27,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-11-28T03:43:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-11-28T03:43:27,153 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:27,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22db69e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:27,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:27,154 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd27826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:27,156 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@151a10cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86c88b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:27,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:27,157 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:27,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:27,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:27,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6976c8ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:27,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:27,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:27,159 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:27,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d4e4d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ec7383, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:27,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:27,160 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:27,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:27,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:27,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5452990e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:27,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:27,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:27,162 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2678b6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25354d98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:27,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:27,163 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c6d39a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:27,165 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:27,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2640dcfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16676eb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:27,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:27,166 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43535eff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:27,168 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bdd519b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4e69c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:27,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:27,169 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5282af16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:27,170 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@271f189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@381f216b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:27,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:27,172 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379b6326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:27,173 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603adf4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325c82fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:27,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:27,175 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a97a4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:27,177 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fcb9b7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f6d2dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:27,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:27,178 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3301d604, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:27,180 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fbcd8a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dcd5d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,38327,-1 for getting cluster id 2024-11-28T03:43:27,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:43:27,181 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68328222-4272-433b-8f44-b2792fc37682' 2024-11-28T03:43:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:43:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68328222-4272-433b-8f44-b2792fc37682" 2024-11-28T03:43:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fa6645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,38327,-1] 2024-11-28T03:43:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:43:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:27,183 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:43:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d0a3d83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:43:27,183 DEBUG [Thread-2723 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:27,183 DEBUG [Thread-2725 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:27,184 DEBUG [Thread-2727 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:27,184 DEBUG [Thread-2730 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:27,184 DEBUG [Thread-2721 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:27,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:27,185 DEBUG [Thread-2723 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:27,185 DEBUG [Thread-2736 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:27,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:27,186 DEBUG [Thread-2738 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:27,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:27,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:27,186 DEBUG [Thread-2734 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:27,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-28T03:43:27,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:27,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:27,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:27,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:27,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:27,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:27,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:27,189 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:43:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:43:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T03:43:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:43:27,190 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:27,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:27,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:43:27,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:43:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:43:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:43:27,193 DEBUG [Thread-2732 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:43:27,200 DEBUG [Thread-2719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:43:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,34671,1732765249447, seqNum=-1] 2024-11-28T03:43:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:43:27,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:27,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:43:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:43:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5] 2024-11-28T03:43:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:43:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:27,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765407225, max timestamp=1732765407231Num uniques 859; 2024-11-28T03:43:27,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765407225, max timestamp=1732765407231Num uniques 859; 2024-11-28T03:43:27,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:43:27,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732765407225, max timestamp=1732765407231Num uniques 859; 2024-11-28T03:43:27,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:43:27,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T03:43:27,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128724f7ba354ec4ae8b64a2fb131891346_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765407201/Put/seqid=0 2024-11-28T03:43:27,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742443_1619 (size=14594) 2024-11-28T03:43:27,242 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:27,246 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128724f7ba354ec4ae8b64a2fb131891346_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128724f7ba354ec4ae8b64a2fb131891346_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:27,247 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/1066cb424aae4d328009f0591633de86, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:27,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/1066cb424aae4d328009f0591633de86 is 175, key is test_row_0/A:col10/1732765407201/Put/seqid=0 2024-11-28T03:43:27,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765467251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765467253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765467255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765467257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765467258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742444_1620 (size=39549) 2024-11-28T03:43:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T03:43:27,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-28T03:43:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765467367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765467367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765467367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765467367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765467367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-28T03:43:27,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:27,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T03:43:27,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765467576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765467576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765467576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765467576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765467577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-28T03:43:27,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:27,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,670 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/1066cb424aae4d328009f0591633de86 2024-11-28T03:43:27,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/74733dae86094cd0be345a322168f040 is 50, key is test_row_0/B:col10/1732765407201/Put/seqid=0 2024-11-28T03:43:27,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742445_1621 (size=12001) 2024-11-28T03:43:27,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/74733dae86094cd0be345a322168f040 2024-11-28T03:43:27,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/58e8eb27df11478f919f3e85425edd84 is 50, key is test_row_0/C:col10/1732765407201/Put/seqid=0 2024-11-28T03:43:27,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742446_1622 (size=12001) 2024-11-28T03:43:27,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-28T03:43:27,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:27,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T03:43:27,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765467886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765467886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765467886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765467887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765467887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:27,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-28T03:43:27,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:27,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:27,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:28,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-28T03:43:28,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:28,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:28,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:28,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:28,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:28,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/58e8eb27df11478f919f3e85425edd84 2024-11-28T03:43:28,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/1066cb424aae4d328009f0591633de86 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/1066cb424aae4d328009f0591633de86 2024-11-28T03:43:28,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/1066cb424aae4d328009f0591633de86, entries=200, sequenceid=14, filesize=38.6 K 2024-11-28T03:43:28,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/74733dae86094cd0be345a322168f040 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/74733dae86094cd0be345a322168f040 2024-11-28T03:43:28,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/74733dae86094cd0be345a322168f040, entries=150, sequenceid=14, filesize=11.7 K 2024-11-28T03:43:28,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/58e8eb27df11478f919f3e85425edd84 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/58e8eb27df11478f919f3e85425edd84 2024-11-28T03:43:28,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/58e8eb27df11478f919f3e85425edd84, entries=150, sequenceid=14, filesize=11.7 K 2024-11-28T03:43:28,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6c2355f8fea173e178470cf1c753f473 in 944ms, sequenceid=14, compaction requested=false 2024-11-28T03:43:28,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:28,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-28T03:43:28,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:28,264 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T03:43:28,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:28,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:28,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:28,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:28,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:28,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:28,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285ca6542cd4ea44de89eb940a4adf6d4b_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765407256/Put/seqid=0 2024-11-28T03:43:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T03:43:28,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742447_1623 (size=12154) 2024-11-28T03:43:28,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:28,339 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285ca6542cd4ea44de89eb940a4adf6d4b_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285ca6542cd4ea44de89eb940a4adf6d4b_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:28,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/a6c8ac006cfc4df48c20012a40db43e3, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:28,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/a6c8ac006cfc4df48c20012a40db43e3 is 175, key is test_row_0/A:col10/1732765407256/Put/seqid=0 2024-11-28T03:43:28,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742448_1624 (size=30955) 2024-11-28T03:43:28,361 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/a6c8ac006cfc4df48c20012a40db43e3 2024-11-28T03:43:28,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/900702700b634daa87f2e9e281b968ff is 50, key is test_row_0/B:col10/1732765407256/Put/seqid=0 2024-11-28T03:43:28,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742449_1625 (size=12001) 2024-11-28T03:43:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:28,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:28,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765468409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765468410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765468411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765468411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765468413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765468517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765468517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765468526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765468526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765468526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765468727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765468727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765468736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765468736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765468737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:28,787 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/900702700b634daa87f2e9e281b968ff 2024-11-28T03:43:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/e99cf85302d045e3a7d49f32db7e3f37 is 50, key is test_row_0/C:col10/1732765407256/Put/seqid=0 2024-11-28T03:43:28,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742450_1626 (size=12001) 2024-11-28T03:43:28,813 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/e99cf85302d045e3a7d49f32db7e3f37 2024-11-28T03:43:28,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/a6c8ac006cfc4df48c20012a40db43e3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a6c8ac006cfc4df48c20012a40db43e3 2024-11-28T03:43:28,822 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a6c8ac006cfc4df48c20012a40db43e3, entries=150, sequenceid=50, filesize=30.2 K 2024-11-28T03:43:28,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/900702700b634daa87f2e9e281b968ff as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/900702700b634daa87f2e9e281b968ff 2024-11-28T03:43:28,833 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/900702700b634daa87f2e9e281b968ff, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T03:43:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/e99cf85302d045e3a7d49f32db7e3f37 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/e99cf85302d045e3a7d49f32db7e3f37 2024-11-28T03:43:28,844 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:43:28,846 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/e99cf85302d045e3a7d49f32db7e3f37, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T03:43:28,847 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6c2355f8fea173e178470cf1c753f473 in 583ms, sequenceid=50, compaction requested=false 2024-11-28T03:43:28,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:28,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:28,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-28T03:43:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-28T03:43:28,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-28T03:43:28,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6590 sec 2024-11-28T03:43:28,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.6720 sec 2024-11-28T03:43:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:29,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T03:43:29,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:29,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:29,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:29,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:29,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:29,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:29,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e90c2e8b1b1c42298f9ed4506c78e974_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765409037/Put/seqid=0 2024-11-28T03:43:29,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742451_1627 (size=17034) 2024-11-28T03:43:29,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765469062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765469062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765469063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765469068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765469069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765469176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765469176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765469176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765469176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765469177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T03:43:29,336 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:29,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-28T03:43:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T03:43:29,339 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:29,339 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:29,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:29,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765469386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765469386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765469387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765469387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765469387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T03:43:29,457 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:29,461 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e90c2e8b1b1c42298f9ed4506c78e974_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e90c2e8b1b1c42298f9ed4506c78e974_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:29,461 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/7414fc845f5b450e8d6a6b4b44fde4d7, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:29,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/7414fc845f5b450e8d6a6b4b44fde4d7 is 175, key is test_row_0/A:col10/1732765409037/Put/seqid=0 2024-11-28T03:43:29,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742452_1628 (size=48139) 2024-11-28T03:43:29,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T03:43:29,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:29,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:29,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:29,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T03:43:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:29,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T03:43:29,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765469696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765469697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765469697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765469697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765469697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:29,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T03:43:29,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:29,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:29,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:29,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,867 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/7414fc845f5b450e8d6a6b4b44fde4d7 2024-11-28T03:43:29,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/0a922a65f1a64c749f57f812c9506f2f is 50, key is test_row_0/B:col10/1732765409037/Put/seqid=0 2024-11-28T03:43:29,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742453_1629 (size=12001) 2024-11-28T03:43:29,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/0a922a65f1a64c749f57f812c9506f2f 2024-11-28T03:43:29,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/6af29c183e8741ac9eb71b447d7de6dd is 50, key is test_row_0/C:col10/1732765409037/Put/seqid=0 2024-11-28T03:43:29,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742454_1630 (size=12001) 2024-11-28T03:43:29,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T03:43:29,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:29,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:29,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:29,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T03:43:30,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T03:43:30,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:30,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:30,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:30,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:30,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:30,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765470206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:30,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765470206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:30,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765470206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:30,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765470207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:30,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765470216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:30,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T03:43:30,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:30,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:30,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:30,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:30,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:30,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/6af29c183e8741ac9eb71b447d7de6dd 2024-11-28T03:43:30,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/7414fc845f5b450e8d6a6b4b44fde4d7 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7414fc845f5b450e8d6a6b4b44fde4d7 2024-11-28T03:43:30,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7414fc845f5b450e8d6a6b4b44fde4d7, entries=250, sequenceid=62, filesize=47.0 K 2024-11-28T03:43:30,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/0a922a65f1a64c749f57f812c9506f2f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/0a922a65f1a64c749f57f812c9506f2f 2024-11-28T03:43:30,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/0a922a65f1a64c749f57f812c9506f2f, entries=150, sequenceid=62, filesize=11.7 K 2024-11-28T03:43:30,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/6af29c183e8741ac9eb71b447d7de6dd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/6af29c183e8741ac9eb71b447d7de6dd 2024-11-28T03:43:30,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/6af29c183e8741ac9eb71b447d7de6dd, entries=150, sequenceid=62, filesize=11.7 K 2024-11-28T03:43:30,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6c2355f8fea173e178470cf1c753f473 in 1279ms, sequenceid=62, compaction requested=true 2024-11-28T03:43:30,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:30,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:30,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:30,319 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:30,319 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:30,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:30,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:30,320 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:30,320 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/B is initiating minor compaction (all files) 2024-11-28T03:43:30,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:30,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:30,320 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/B in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:30,321 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/74733dae86094cd0be345a322168f040, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/900702700b634daa87f2e9e281b968ff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/0a922a65f1a64c749f57f812c9506f2f] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=35.2 K 2024-11-28T03:43:30,321 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:30,321 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/A is initiating minor compaction (all files) 2024-11-28T03:43:30,321 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/A in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:30,321 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/1066cb424aae4d328009f0591633de86, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a6c8ac006cfc4df48c20012a40db43e3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7414fc845f5b450e8d6a6b4b44fde4d7] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=115.9 K 2024-11-28T03:43:30,321 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:30,321 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/1066cb424aae4d328009f0591633de86, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a6c8ac006cfc4df48c20012a40db43e3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7414fc845f5b450e8d6a6b4b44fde4d7] 2024-11-28T03:43:30,321 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 74733dae86094cd0be345a322168f040, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732765407200 2024-11-28T03:43:30,322 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1066cb424aae4d328009f0591633de86, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732765407192 2024-11-28T03:43:30,322 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 900702700b634daa87f2e9e281b968ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732765407251 2024-11-28T03:43:30,322 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a922a65f1a64c749f57f812c9506f2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732765408409 2024-11-28T03:43:30,322 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c8ac006cfc4df48c20012a40db43e3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732765407251 2024-11-28T03:43:30,322 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7414fc845f5b450e8d6a6b4b44fde4d7, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732765408409 2024-11-28T03:43:30,329 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#B#compaction#546 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:30,329 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:30,330 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/582811fc7cd842e8bd3b5a5d565a48ec is 50, key is test_row_0/B:col10/1732765409037/Put/seqid=0 2024-11-28T03:43:30,333 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411288dfd08181ba4463d8b4ab2716ea5b27b_6c2355f8fea173e178470cf1c753f473 store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:30,335 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411288dfd08181ba4463d8b4ab2716ea5b27b_6c2355f8fea173e178470cf1c753f473, store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:30,335 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288dfd08181ba4463d8b4ab2716ea5b27b_6c2355f8fea173e178470cf1c753f473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:30,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742455_1631 (size=12104) 2024-11-28T03:43:30,348 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/582811fc7cd842e8bd3b5a5d565a48ec as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/582811fc7cd842e8bd3b5a5d565a48ec 2024-11-28T03:43:30,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742456_1632 (size=4469) 2024-11-28T03:43:30,355 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#A#compaction#547 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:30,356 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/8b8637c3175b48b9ae033d48c26028df is 175, key is test_row_0/A:col10/1732765409037/Put/seqid=0 2024-11-28T03:43:30,356 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/B of 6c2355f8fea173e178470cf1c753f473 into 582811fc7cd842e8bd3b5a5d565a48ec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:30,356 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:30,356 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/B, priority=13, startTime=1732765410319; duration=0sec 2024-11-28T03:43:30,356 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:30,356 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:B 2024-11-28T03:43:30,356 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:30,359 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:30,359 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/C is initiating minor compaction (all files) 2024-11-28T03:43:30,359 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/C in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:30,359 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/58e8eb27df11478f919f3e85425edd84, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/e99cf85302d045e3a7d49f32db7e3f37, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/6af29c183e8741ac9eb71b447d7de6dd] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=35.2 K 2024-11-28T03:43:30,360 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e8eb27df11478f919f3e85425edd84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732765407200 2024-11-28T03:43:30,360 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e99cf85302d045e3a7d49f32db7e3f37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732765407251 2024-11-28T03:43:30,361 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6af29c183e8741ac9eb71b447d7de6dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732765408409 2024-11-28T03:43:30,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742457_1633 (size=31058) 2024-11-28T03:43:30,368 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/8b8637c3175b48b9ae033d48c26028df as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8b8637c3175b48b9ae033d48c26028df 2024-11-28T03:43:30,370 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#C#compaction#548 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:30,371 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/5f70d40a7e8e497788d6654919652a10 is 50, key is test_row_0/C:col10/1732765409037/Put/seqid=0 2024-11-28T03:43:30,375 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/A of 6c2355f8fea173e178470cf1c753f473 into 8b8637c3175b48b9ae033d48c26028df(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:30,375 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:30,375 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/A, priority=13, startTime=1732765410318; duration=0sec 2024-11-28T03:43:30,375 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:30,375 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:A 2024-11-28T03:43:30,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742458_1634 (size=12104) 2024-11-28T03:43:30,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T03:43:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:30,411 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T03:43:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:30,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:30,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287cdef6c16e61401c933107f56bd03990_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765409060/Put/seqid=0 2024-11-28T03:43:30,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742459_1635 (size=12154) 2024-11-28T03:43:30,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,437 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287cdef6c16e61401c933107f56bd03990_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287cdef6c16e61401c933107f56bd03990_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:30,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d is 175, key is test_row_0/A:col10/1732765409060/Put/seqid=0 2024-11-28T03:43:30,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742460_1636 (size=30955) 2024-11-28T03:43:30,452 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d 2024-11-28T03:43:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/68a08d6b14704df8b39c34dfa8f4d00c is 50, key is test_row_0/B:col10/1732765409060/Put/seqid=0 2024-11-28T03:43:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T03:43:30,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742461_1637 (size=12001) 2024-11-28T03:43:30,501 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/68a08d6b14704df8b39c34dfa8f4d00c 2024-11-28T03:43:30,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/3b371558478848e3a3ee49de80713486 is 50, key is test_row_0/C:col10/1732765409060/Put/seqid=0 2024-11-28T03:43:30,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742462_1638 (size=12001) 2024-11-28T03:43:30,549 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/3b371558478848e3a3ee49de80713486 2024-11-28T03:43:30,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d 2024-11-28T03:43:30,560 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d, entries=150, sequenceid=84, filesize=30.2 K 2024-11-28T03:43:30,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/68a08d6b14704df8b39c34dfa8f4d00c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/68a08d6b14704df8b39c34dfa8f4d00c 2024-11-28T03:43:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,564 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/68a08d6b14704df8b39c34dfa8f4d00c, entries=150, sequenceid=84, filesize=11.7 K 2024-11-28T03:43:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/3b371558478848e3a3ee49de80713486 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3b371558478848e3a3ee49de80713486 2024-11-28T03:43:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,570 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3b371558478848e3a3ee49de80713486, entries=150, sequenceid=84, filesize=11.7 K 2024-11-28T03:43:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,571 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 6c2355f8fea173e178470cf1c753f473 in 159ms, sequenceid=84, compaction requested=false 2024-11-28T03:43:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-28T03:43:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-28T03:43:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-28T03:43:30,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2330 sec 2024-11-28T03:43:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.2380 sec 2024-11-28T03:43:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,799 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/5f70d40a7e8e497788d6654919652a10 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5f70d40a7e8e497788d6654919652a10 2024-11-28T03:43:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,805 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/C of 6c2355f8fea173e178470cf1c753f473 into 5f70d40a7e8e497788d6654919652a10(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:30,805 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:30,805 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/C, priority=13, startTime=1732765410320; duration=0sec 2024-11-28T03:43:30,805 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:30,805 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:C 2024-11-28T03:43:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T03:43:31,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:31,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:31,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:31,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:31,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:31,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128603e7e3e09d74751a597e514fcfb0484_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765411259/Put/seqid=0 2024-11-28T03:43:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742463_1639 (size=19474) 2024-11-28T03:43:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30175, totalHeapSize=68776, min timestamp=1732765411279, max timestamp=1732765411311Num uniques 859; 2024-11-28T03:43:31,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30175, totalHeapSize=68776, min timestamp=1732765411279, max timestamp=1732765411311Num uniques 859; 2024-11-28T03:43:31,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=30175, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-11-28T03:43:31,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=30175, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-11-28T03:43:31,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30175, totalHeapSize=68776, min timestamp=1732765411279, max timestamp=1732765411311Num uniques 859; 2024-11-28T03:43:31,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3a546af21488:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=30175, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-11-28T03:43:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765471321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765471322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765471323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765471326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765471327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765471436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765471436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765471436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765471436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765471436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T03:43:31,486 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:31,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-28T03:43:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T03:43:31,489 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:31,489 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:31,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T03:43:31,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T03:43:31,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:31,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:31,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:31,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:31,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:31,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765471646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765471647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765471647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765471647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765471647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,673 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:31,676 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128603e7e3e09d74751a597e514fcfb0484_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128603e7e3e09d74751a597e514fcfb0484_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:31,677 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/b122a1425b9f4ea7b5c07527b1875bbd, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:31,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/b122a1425b9f4ea7b5c07527b1875bbd is 175, key is test_row_0/A:col10/1732765411259/Put/seqid=0 2024-11-28T03:43:31,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742464_1640 (size=56729) 2024-11-28T03:43:31,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T03:43:31,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:31,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:31,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:31,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:31,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T03:43:31,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T03:43:31,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:31,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:31,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:31,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:31,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:31,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765471956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765471956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765471957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765471957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765471957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,098 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/b122a1425b9f4ea7b5c07527b1875bbd 2024-11-28T03:43:32,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T03:43:32,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:32,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:32,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:32,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:32,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:32,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/8278ffa10d2d4503930fbe22c776fe0d is 50, key is test_row_0/B:col10/1732765411259/Put/seqid=0 2024-11-28T03:43:32,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742465_1641 (size=9657) 2024-11-28T03:43:32,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/8278ffa10d2d4503930fbe22c776fe0d 2024-11-28T03:43:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T03:43:32,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/d638c4c535a84ee29c9965130510433a is 50, key is test_row_0/C:col10/1732765411259/Put/seqid=0 2024-11-28T03:43:32,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742466_1642 (size=9657) 2024-11-28T03:43:32,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/d638c4c535a84ee29c9965130510433a 2024-11-28T03:43:32,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/b122a1425b9f4ea7b5c07527b1875bbd as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b122a1425b9f4ea7b5c07527b1875bbd 2024-11-28T03:43:32,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b122a1425b9f4ea7b5c07527b1875bbd, entries=300, sequenceid=95, filesize=55.4 K 2024-11-28T03:43:32,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/8278ffa10d2d4503930fbe22c776fe0d as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/8278ffa10d2d4503930fbe22c776fe0d 2024-11-28T03:43:32,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/8278ffa10d2d4503930fbe22c776fe0d, entries=100, sequenceid=95, filesize=9.4 K 2024-11-28T03:43:32,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/d638c4c535a84ee29c9965130510433a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/d638c4c535a84ee29c9965130510433a 2024-11-28T03:43:32,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/d638c4c535a84ee29c9965130510433a, entries=100, sequenceid=95, filesize=9.4 K 2024-11-28T03:43:32,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 6c2355f8fea173e178470cf1c753f473 in 885ms, sequenceid=95, compaction requested=true 2024-11-28T03:43:32,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:32,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:32,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:32,145 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:32,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:32,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:32,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:32,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:32,145 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:32,146 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118742 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:32,147 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/A is initiating minor compaction (all files) 2024-11-28T03:43:32,147 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/A in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:32,147 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8b8637c3175b48b9ae033d48c26028df, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b122a1425b9f4ea7b5c07527b1875bbd] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=116.0 K 2024-11-28T03:43:32,147 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:32,147 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8b8637c3175b48b9ae033d48c26028df, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b122a1425b9f4ea7b5c07527b1875bbd] 2024-11-28T03:43:32,147 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:32,147 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b8637c3175b48b9ae033d48c26028df, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732765408409 2024-11-28T03:43:32,147 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/B is initiating minor compaction (all files) 2024-11-28T03:43:32,147 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/B in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:32,147 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/582811fc7cd842e8bd3b5a5d565a48ec, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/68a08d6b14704df8b39c34dfa8f4d00c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/8278ffa10d2d4503930fbe22c776fe0d] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=33.0 K 2024-11-28T03:43:32,148 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 582811fc7cd842e8bd3b5a5d565a48ec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732765408409 2024-11-28T03:43:32,148 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ba5a1cef8ce4b3dbc9fc2bd40c3734d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732765409060 2024-11-28T03:43:32,149 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b122a1425b9f4ea7b5c07527b1875bbd, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732765411237 2024-11-28T03:43:32,149 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 68a08d6b14704df8b39c34dfa8f4d00c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732765409060 2024-11-28T03:43:32,149 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8278ffa10d2d4503930fbe22c776fe0d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732765411258 2024-11-28T03:43:32,158 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#B#compaction#555 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:32,158 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:32,158 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/90916ab6b0694e11bafb9a8dbdc291df is 50, key is test_row_0/B:col10/1732765411259/Put/seqid=0 2024-11-28T03:43:32,164 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112865bfcfa1824a407bb7747507a7afa0f3_6c2355f8fea173e178470cf1c753f473 store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:32,166 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112865bfcfa1824a407bb7747507a7afa0f3_6c2355f8fea173e178470cf1c753f473, store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:32,166 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112865bfcfa1824a407bb7747507a7afa0f3_6c2355f8fea173e178470cf1c753f473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:32,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742467_1643 (size=12207) 2024-11-28T03:43:32,175 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/90916ab6b0694e11bafb9a8dbdc291df as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/90916ab6b0694e11bafb9a8dbdc291df 2024-11-28T03:43:32,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742468_1644 (size=4469) 2024-11-28T03:43:32,177 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#A#compaction#556 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:32,177 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/e072b19124d84e1394a0e80de845f091 is 175, key is test_row_0/A:col10/1732765411259/Put/seqid=0 2024-11-28T03:43:32,181 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/B of 6c2355f8fea173e178470cf1c753f473 into 90916ab6b0694e11bafb9a8dbdc291df(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:32,181 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:32,181 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/B, priority=13, startTime=1732765412145; duration=0sec 2024-11-28T03:43:32,181 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:32,181 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:B 2024-11-28T03:43:32,181 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:32,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742469_1645 (size=31268) 2024-11-28T03:43:32,182 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:32,182 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/C is initiating minor compaction (all files) 2024-11-28T03:43:32,183 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/C in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:32,183 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5f70d40a7e8e497788d6654919652a10, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3b371558478848e3a3ee49de80713486, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/d638c4c535a84ee29c9965130510433a] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=33.0 K 2024-11-28T03:43:32,183 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f70d40a7e8e497788d6654919652a10, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732765408409 2024-11-28T03:43:32,183 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b371558478848e3a3ee49de80713486, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732765409060 2024-11-28T03:43:32,183 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting d638c4c535a84ee29c9965130510433a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732765411258 2024-11-28T03:43:32,189 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#C#compaction#557 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:32,189 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/53230c6c622342769423b4ac0120a17a is 50, key is test_row_0/C:col10/1732765411259/Put/seqid=0 2024-11-28T03:43:32,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742470_1646 (size=12207) 2024-11-28T03:43:32,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T03:43:32,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:32,256 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=147.60 KB heapSize=377.95 KB 2024-11-28T03:43:32,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:32,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:32,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:32,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:32,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:32,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T03:43:32,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112881b62f39d8d7490f9f95427ba21b0392_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765411324/Put/seqid=0 2024-11-28T03:43:32,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742471_1647 (size=12154) 2024-11-28T03:43:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:32,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765472472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765472473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765472479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765472480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765472481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,586 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/e072b19124d84e1394a0e80de845f091 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/e072b19124d84e1394a0e80de845f091 2024-11-28T03:43:32,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765472586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765472586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,590 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/A of 6c2355f8fea173e178470cf1c753f473 into e072b19124d84e1394a0e80de845f091(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:32,590 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:32,590 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/A, priority=13, startTime=1732765412145; duration=0sec 2024-11-28T03:43:32,590 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:32,590 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:A 2024-11-28T03:43:32,596 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/53230c6c622342769423b4ac0120a17a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/53230c6c622342769423b4ac0120a17a 2024-11-28T03:43:32,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765472596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765472596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765472596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,600 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/C of 6c2355f8fea173e178470cf1c753f473 into 53230c6c622342769423b4ac0120a17a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:32,600 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:32,600 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/C, priority=13, startTime=1732765412145; duration=0sec 2024-11-28T03:43:32,600 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:32,600 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:C 2024-11-28T03:43:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T03:43:32,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:32,671 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112881b62f39d8d7490f9f95427ba21b0392_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112881b62f39d8d7490f9f95427ba21b0392_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/4e442ed4c6514af99da517261269b4ff, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/4e442ed4c6514af99da517261269b4ff is 175, key is test_row_0/A:col10/1732765411324/Put/seqid=0 2024-11-28T03:43:32,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742472_1648 (size=30955) 2024-11-28T03:43:32,676 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=125, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/4e442ed4c6514af99da517261269b4ff 2024-11-28T03:43:32,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/61e115abf669425198991b37def619c0 is 50, key is test_row_0/B:col10/1732765411324/Put/seqid=0 2024-11-28T03:43:32,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742473_1649 (size=12001) 2024-11-28T03:43:32,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765472796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765472796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765472806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765472807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765472807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,087 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/61e115abf669425198991b37def619c0 2024-11-28T03:43:33,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/05b86844589a4bccb64bc8512452d141 is 50, key is test_row_0/C:col10/1732765411324/Put/seqid=0 2024-11-28T03:43:33,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742474_1650 (size=12001) 2024-11-28T03:43:33,100 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/05b86844589a4bccb64bc8512452d141 2024-11-28T03:43:33,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/4e442ed4c6514af99da517261269b4ff as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4e442ed4c6514af99da517261269b4ff 2024-11-28T03:43:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765473106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,109 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4e442ed4c6514af99da517261269b4ff, entries=150, sequenceid=125, filesize=30.2 K 2024-11-28T03:43:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765473106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/61e115abf669425198991b37def619c0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/61e115abf669425198991b37def619c0 2024-11-28T03:43:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,113 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/61e115abf669425198991b37def619c0, entries=150, sequenceid=125, filesize=11.7 K 2024-11-28T03:43:33,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/05b86844589a4bccb64bc8512452d141 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/05b86844589a4bccb64bc8512452d141 2024-11-28T03:43:33,118 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/05b86844589a4bccb64bc8512452d141, entries=150, sequenceid=125, filesize=11.7 K 2024-11-28T03:43:33,119 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~377.91 KB/386976, currentSize=53.67 KB/54960 for 6c2355f8fea173e178470cf1c753f473 in 863ms, sequenceid=125, compaction requested=false 2024-11-28T03:43:33,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:33,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:33,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-28T03:43:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-28T03:43:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:33,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T03:43:33,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:33,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:33,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:33,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:33,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:33,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:33,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-28T03:43:33,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-11-28T03:43:33,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.6390 sec 2024-11-28T03:43:33,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d52cb557c30f4146bc789cede5296f7a_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765412469/Put/seqid=0 2024-11-28T03:43:33,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742475_1651 (size=12304) 2024-11-28T03:43:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765473159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765473161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765473160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765473277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765473277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765473277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765473487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765473487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765473487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,559 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:33,564 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d52cb557c30f4146bc789cede5296f7a_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d52cb557c30f4146bc789cede5296f7a_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:33,566 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/d9f98e686e654e46af88b07ac88c51e0, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:33,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/d9f98e686e654e46af88b07ac88c51e0 is 175, key is test_row_0/A:col10/1732765412469/Put/seqid=0 2024-11-28T03:43:33,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742476_1652 (size=31105) 2024-11-28T03:43:33,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765473616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765473626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T03:43:33,637 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:33,638 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-28T03:43:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T03:43:33,640 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:33,641 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:33,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T03:43:33,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-28T03:43:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:33,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:33,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765473797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765473797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765473797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:33,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-28T03:43:33,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:33,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:33,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:33,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:33,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T03:43:33,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/d9f98e686e654e46af88b07ac88c51e0 2024-11-28T03:43:33,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/2b9b0c16ec054f85ab2bb1f3fef144eb is 50, key is test_row_0/B:col10/1732765412469/Put/seqid=0 2024-11-28T03:43:33,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742477_1653 (size=12151) 2024-11-28T03:43:33,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/2b9b0c16ec054f85ab2bb1f3fef144eb 2024-11-28T03:43:33,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/111962b1c2854cb29948d9dc0b5b5a5c is 50, key is test_row_0/C:col10/1732765412469/Put/seqid=0 2024-11-28T03:43:33,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742478_1654 (size=12151) 2024-11-28T03:43:34,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-28T03:43:34,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:34,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:34,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:34,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-28T03:43:34,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:34,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:34,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T03:43:34,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765474306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:34,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765474307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:34,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765474307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:34,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/111962b1c2854cb29948d9dc0b5b5a5c 2024-11-28T03:43:34,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/d9f98e686e654e46af88b07ac88c51e0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/d9f98e686e654e46af88b07ac88c51e0 2024-11-28T03:43:34,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/d9f98e686e654e46af88b07ac88c51e0, entries=150, sequenceid=141, filesize=30.4 K 2024-11-28T03:43:34,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/2b9b0c16ec054f85ab2bb1f3fef144eb as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/2b9b0c16ec054f85ab2bb1f3fef144eb 2024-11-28T03:43:34,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/2b9b0c16ec054f85ab2bb1f3fef144eb, entries=150, sequenceid=141, filesize=11.9 K 2024-11-28T03:43:34,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/111962b1c2854cb29948d9dc0b5b5a5c as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/111962b1c2854cb29948d9dc0b5b5a5c 2024-11-28T03:43:34,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-28T03:43:34,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:34,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:34,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:34,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/111962b1c2854cb29948d9dc0b5b5a5c, entries=150, sequenceid=141, filesize=11.9 K 2024-11-28T03:43:34,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 6c2355f8fea173e178470cf1c753f473 in 1294ms, sequenceid=141, compaction requested=true 2024-11-28T03:43:34,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:34,416 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:34,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:34,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:34,417 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:34,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:34,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:34,417 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93328 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:34,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:34,417 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/A is initiating minor compaction (all files) 2024-11-28T03:43:34,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:34,417 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/A in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,417 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/e072b19124d84e1394a0e80de845f091, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4e442ed4c6514af99da517261269b4ff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/d9f98e686e654e46af88b07ac88c51e0] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=91.1 K 2024-11-28T03:43:34,417 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,417 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/e072b19124d84e1394a0e80de845f091, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4e442ed4c6514af99da517261269b4ff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/d9f98e686e654e46af88b07ac88c51e0] 2024-11-28T03:43:34,418 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e072b19124d84e1394a0e80de845f091, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732765409061 2024-11-28T03:43:34,418 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:34,418 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/B is initiating minor compaction (all files) 2024-11-28T03:43:34,418 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/B in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,419 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/90916ab6b0694e11bafb9a8dbdc291df, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/61e115abf669425198991b37def619c0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/2b9b0c16ec054f85ab2bb1f3fef144eb] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=35.5 K 2024-11-28T03:43:34,419 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e442ed4c6514af99da517261269b4ff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732765411322 2024-11-28T03:43:34,419 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 90916ab6b0694e11bafb9a8dbdc291df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732765409061 2024-11-28T03:43:34,419 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9f98e686e654e46af88b07ac88c51e0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732765412469 2024-11-28T03:43:34,420 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 61e115abf669425198991b37def619c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732765411322 2024-11-28T03:43:34,420 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9b0c16ec054f85ab2bb1f3fef144eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732765412469 2024-11-28T03:43:34,425 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:34,436 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128463dbf0ba9404a45bb365bb8163b7543_6c2355f8fea173e178470cf1c753f473 store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:34,438 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#B#compaction#565 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:34,438 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128463dbf0ba9404a45bb365bb8163b7543_6c2355f8fea173e178470cf1c753f473, store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:34,439 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128463dbf0ba9404a45bb365bb8163b7543_6c2355f8fea173e178470cf1c753f473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:34,439 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/415a66a9c16f45dfb085a863053cfa6e is 50, key is test_row_0/B:col10/1732765412469/Put/seqid=0 2024-11-28T03:43:34,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742479_1655 (size=4469) 2024-11-28T03:43:34,457 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#A#compaction#564 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:34,458 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/39c40a6d1a2144b089fcc25c0a18cf08 is 175, key is test_row_0/A:col10/1732765412469/Put/seqid=0 2024-11-28T03:43:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742481_1657 (size=31413) 2024-11-28T03:43:34,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742480_1656 (size=12459) 2024-11-28T03:43:34,490 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/39c40a6d1a2144b089fcc25c0a18cf08 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/39c40a6d1a2144b089fcc25c0a18cf08 2024-11-28T03:43:34,495 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/A of 6c2355f8fea173e178470cf1c753f473 into 39c40a6d1a2144b089fcc25c0a18cf08(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:34,495 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:34,495 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/A, priority=13, startTime=1732765414416; duration=0sec 2024-11-28T03:43:34,495 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:34,495 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:A 2024-11-28T03:43:34,495 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:34,496 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:34,497 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/C is initiating minor compaction (all files) 2024-11-28T03:43:34,497 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/C in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,497 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/53230c6c622342769423b4ac0120a17a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/05b86844589a4bccb64bc8512452d141, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/111962b1c2854cb29948d9dc0b5b5a5c] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=35.5 K 2024-11-28T03:43:34,498 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53230c6c622342769423b4ac0120a17a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732765409061 2024-11-28T03:43:34,498 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05b86844589a4bccb64bc8512452d141, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732765411322 2024-11-28T03:43:34,498 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 111962b1c2854cb29948d9dc0b5b5a5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732765412469 2024-11-28T03:43:34,504 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#C#compaction#566 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:34,505 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/920e9a294be64faa95ce64cff5b605e2 is 50, key is test_row_0/C:col10/1732765412469/Put/seqid=0 2024-11-28T03:43:34,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742482_1658 (size=12459) 2024-11-28T03:43:34,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-28T03:43:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:34,564 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:43:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:34,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112859d3172e3a224083807f83249e0bf681_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765413157/Put/seqid=0 2024-11-28T03:43:34,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742483_1659 (size=12304) 2024-11-28T03:43:34,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:34,584 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112859d3172e3a224083807f83249e0bf681_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112859d3172e3a224083807f83249e0bf681_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:34,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/7a558c5fed6b4b058c6a5c9cdeb57580, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:34,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/7a558c5fed6b4b058c6a5c9cdeb57580 is 175, key is test_row_0/A:col10/1732765413157/Put/seqid=0 2024-11-28T03:43:34,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742484_1660 (size=31105) 2024-11-28T03:43:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:34,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:34,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765474664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:34,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765474664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T03:43:34,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765474776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:34,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765474776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:34,891 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/415a66a9c16f45dfb085a863053cfa6e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/415a66a9c16f45dfb085a863053cfa6e 2024-11-28T03:43:34,895 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/B of 6c2355f8fea173e178470cf1c753f473 into 415a66a9c16f45dfb085a863053cfa6e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:34,895 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:34,895 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/B, priority=13, startTime=1732765414416; duration=0sec 2024-11-28T03:43:34,895 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:34,895 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:B 2024-11-28T03:43:34,922 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/920e9a294be64faa95ce64cff5b605e2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/920e9a294be64faa95ce64cff5b605e2 2024-11-28T03:43:34,926 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/C of 6c2355f8fea173e178470cf1c753f473 into 920e9a294be64faa95ce64cff5b605e2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:34,927 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:34,927 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/C, priority=13, startTime=1732765414417; duration=0sec 2024-11-28T03:43:34,927 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:34,927 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:C 2024-11-28T03:43:34,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765474986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:34,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765474986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:34,994 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/7a558c5fed6b4b058c6a5c9cdeb57580 2024-11-28T03:43:35,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/7a1187aa1fa2459baa585565c9111cbb is 50, key is test_row_0/B:col10/1732765413157/Put/seqid=0 2024-11-28T03:43:35,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742485_1661 (size=12151) 2024-11-28T03:43:35,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765475296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:35,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765475297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:35,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765475316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:35,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765475316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:35,318 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:35,319 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_0' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:35,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765475326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:35,327 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:35,405 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/7a1187aa1fa2459baa585565c9111cbb 2024-11-28T03:43:35,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/27182b8363d44794bac808f09b82ae42 is 50, key is test_row_0/C:col10/1732765413157/Put/seqid=0 2024-11-28T03:43:35,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742486_1662 (size=12151) 2024-11-28T03:43:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T03:43:35,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765475806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765475806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:35,816 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/27182b8363d44794bac808f09b82ae42 2024-11-28T03:43:35,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/7a558c5fed6b4b058c6a5c9cdeb57580 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7a558c5fed6b4b058c6a5c9cdeb57580 2024-11-28T03:43:35,823 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7a558c5fed6b4b058c6a5c9cdeb57580, entries=150, sequenceid=161, filesize=30.4 K 2024-11-28T03:43:35,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/7a1187aa1fa2459baa585565c9111cbb as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/7a1187aa1fa2459baa585565c9111cbb 2024-11-28T03:43:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,827 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/7a1187aa1fa2459baa585565c9111cbb, entries=150, sequenceid=161, filesize=11.9 K 2024-11-28T03:43:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/27182b8363d44794bac808f09b82ae42 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/27182b8363d44794bac808f09b82ae42 2024-11-28T03:43:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,834 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/27182b8363d44794bac808f09b82ae42, entries=150, sequenceid=161, filesize=11.9 K 2024-11-28T03:43:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,834 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6c2355f8fea173e178470cf1c753f473 in 1270ms, sequenceid=161, compaction requested=false 2024-11-28T03:43:35,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:35,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-28T03:43:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-28T03:43:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-28T03:43:35,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1950 sec 2024-11-28T03:43:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 2.2010 sec 2024-11-28T03:43:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:36,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:43:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112852b751b646cd4ccfb57067a64c75d92a_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765414663/Put/seqid=0 2024-11-28T03:43:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742487_1663 (size=12304) 2024-11-28T03:43:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:36,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765476862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:36,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:36,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:36,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:36,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765476867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765476976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:36,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765476976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:37,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765477187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765477187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:37,232 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:37,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112852b751b646cd4ccfb57067a64c75d92a_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112852b751b646cd4ccfb57067a64c75d92a_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:37,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/991304bc8b1d43249f76f319d27743b9, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:37,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/991304bc8b1d43249f76f319d27743b9 is 175, key is test_row_0/A:col10/1732765414663/Put/seqid=0 2024-11-28T03:43:37,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742488_1664 (size=31105) 2024-11-28T03:43:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765477326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:37,330 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_0' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:37,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765477336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:37,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765477336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:37,340 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,341 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:37,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765477496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:37,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765477496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:37,641 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/991304bc8b1d43249f76f319d27743b9 2024-11-28T03:43:37,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/71c7649ad2354b6c8881c92832eea5d8 is 50, key is test_row_0/B:col10/1732765414663/Put/seqid=0 2024-11-28T03:43:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742489_1665 (size=12151) 2024-11-28T03:43:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T03:43:37,807 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:37,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-28T03:43:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T03:43:37,810 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:37,811 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:37,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T03:43:37,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-28T03:43:37,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:37,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:37,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:37,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:37,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:38,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765478006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:38,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765478006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:38,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/71c7649ad2354b6c8881c92832eea5d8 2024-11-28T03:43:38,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/3c0400774b274225a1c0eccb99fa3fa2 is 50, key is test_row_0/C:col10/1732765414663/Put/seqid=0 2024-11-28T03:43:38,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742490_1666 (size=12151) 2024-11-28T03:43:38,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-28T03:43:38,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:38,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:38,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T03:43:38,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-28T03:43:38,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:38,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:38,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:38,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-28T03:43:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T03:43:38,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/3c0400774b274225a1c0eccb99fa3fa2 2024-11-28T03:43:38,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/991304bc8b1d43249f76f319d27743b9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/991304bc8b1d43249f76f319d27743b9 2024-11-28T03:43:38,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/991304bc8b1d43249f76f319d27743b9, entries=150, sequenceid=177, filesize=30.4 K 2024-11-28T03:43:38,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/71c7649ad2354b6c8881c92832eea5d8 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/71c7649ad2354b6c8881c92832eea5d8 2024-11-28T03:43:38,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/71c7649ad2354b6c8881c92832eea5d8, entries=150, sequenceid=177, filesize=11.9 K 2024-11-28T03:43:38,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/3c0400774b274225a1c0eccb99fa3fa2 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3c0400774b274225a1c0eccb99fa3fa2 2024-11-28T03:43:38,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3c0400774b274225a1c0eccb99fa3fa2, entries=150, sequenceid=177, filesize=11.9 K 2024-11-28T03:43:38,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6c2355f8fea173e178470cf1c753f473 in 1670ms, sequenceid=177, compaction requested=true 2024-11-28T03:43:38,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:38,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:38,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:38,489 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:38,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:38,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:38,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:38,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:38,489 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:38,490 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:38,490 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:38,490 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/B is initiating minor compaction (all files) 2024-11-28T03:43:38,490 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/A is initiating minor compaction (all files) 2024-11-28T03:43:38,490 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/A in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,490 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/B in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,490 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/39c40a6d1a2144b089fcc25c0a18cf08, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7a558c5fed6b4b058c6a5c9cdeb57580, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/991304bc8b1d43249f76f319d27743b9] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=91.4 K 2024-11-28T03:43:38,490 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/415a66a9c16f45dfb085a863053cfa6e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/7a1187aa1fa2459baa585565c9111cbb, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/71c7649ad2354b6c8881c92832eea5d8] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=35.9 K 2024-11-28T03:43:38,490 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,490 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/39c40a6d1a2144b089fcc25c0a18cf08, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7a558c5fed6b4b058c6a5c9cdeb57580, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/991304bc8b1d43249f76f319d27743b9] 2024-11-28T03:43:38,491 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 415a66a9c16f45dfb085a863053cfa6e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732765412469 2024-11-28T03:43:38,491 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39c40a6d1a2144b089fcc25c0a18cf08, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732765412469 2024-11-28T03:43:38,491 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a1187aa1fa2459baa585565c9111cbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732765413149 2024-11-28T03:43:38,491 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a558c5fed6b4b058c6a5c9cdeb57580, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732765413149 2024-11-28T03:43:38,491 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 71c7649ad2354b6c8881c92832eea5d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732765414663 2024-11-28T03:43:38,491 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 991304bc8b1d43249f76f319d27743b9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732765414663 2024-11-28T03:43:38,497 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:38,498 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#B#compaction#573 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:38,498 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411281e2dc2d5649f4d288afc3c621efc90b9_6c2355f8fea173e178470cf1c753f473 store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:38,499 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/bd3ffa67f9be4f0bb6caa01ad8790c75 is 50, key is test_row_0/B:col10/1732765414663/Put/seqid=0 2024-11-28T03:43:38,500 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411281e2dc2d5649f4d288afc3c621efc90b9_6c2355f8fea173e178470cf1c753f473, store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:38,501 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281e2dc2d5649f4d288afc3c621efc90b9_6c2355f8fea173e178470cf1c753f473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:38,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742491_1667 (size=12561) 2024-11-28T03:43:38,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742492_1668 (size=4469) 2024-11-28T03:43:38,510 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#A#compaction#574 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:38,511 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/fa129e3df3c84a4bb72446eb2e3e193f is 175, key is test_row_0/A:col10/1732765414663/Put/seqid=0 2024-11-28T03:43:38,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742493_1669 (size=31515) 2024-11-28T03:43:38,521 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/fa129e3df3c84a4bb72446eb2e3e193f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/fa129e3df3c84a4bb72446eb2e3e193f 2024-11-28T03:43:38,525 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/A of 6c2355f8fea173e178470cf1c753f473 into fa129e3df3c84a4bb72446eb2e3e193f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:38,525 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:38,525 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/A, priority=13, startTime=1732765418489; duration=0sec 2024-11-28T03:43:38,525 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:38,525 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:A 2024-11-28T03:43:38,525 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T03:43:38,526 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T03:43:38,526 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/C is initiating minor compaction (all files) 2024-11-28T03:43:38,526 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/C in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,527 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/920e9a294be64faa95ce64cff5b605e2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/27182b8363d44794bac808f09b82ae42, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3c0400774b274225a1c0eccb99fa3fa2] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=35.9 K 2024-11-28T03:43:38,527 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 920e9a294be64faa95ce64cff5b605e2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732765412469 2024-11-28T03:43:38,527 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27182b8363d44794bac808f09b82ae42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732765413149 2024-11-28T03:43:38,527 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c0400774b274225a1c0eccb99fa3fa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732765414663 2024-11-28T03:43:38,534 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#C#compaction#575 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:38,534 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/9f2f284ef1fd4df990d281a3d68e343a is 50, key is test_row_0/C:col10/1732765414663/Put/seqid=0 2024-11-28T03:43:38,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742494_1670 (size=12561) 2024-11-28T03:43:38,543 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/9f2f284ef1fd4df990d281a3d68e343a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/9f2f284ef1fd4df990d281a3d68e343a 2024-11-28T03:43:38,547 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/C of 6c2355f8fea173e178470cf1c753f473 into 9f2f284ef1fd4df990d281a3d68e343a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:38,548 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:38,548 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/C, priority=13, startTime=1732765418489; duration=0sec 2024-11-28T03:43:38,548 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:38,548 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:C 2024-11-28T03:43:38,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-28T03:43:38,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:38,576 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:43:38,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:38,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:38,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:38,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:38,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:38,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:38,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ec7cbf15fc454bc0824ea0d1669c7d2f_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765416866/Put/seqid=0 2024-11-28T03:43:38,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742495_1671 (size=12304) 2024-11-28T03:43:38,911 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/bd3ffa67f9be4f0bb6caa01ad8790c75 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/bd3ffa67f9be4f0bb6caa01ad8790c75 2024-11-28T03:43:38,920 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/B of 6c2355f8fea173e178470cf1c753f473 into bd3ffa67f9be4f0bb6caa01ad8790c75(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:38,920 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:38,920 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/B, priority=13, startTime=1732765418489; duration=0sec 2024-11-28T03:43:38,920 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:38,920 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:B 2024-11-28T03:43:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T03:43:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:38,991 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ec7cbf15fc454bc0824ea0d1669c7d2f_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ec7cbf15fc454bc0824ea0d1669c7d2f_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:38,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/f57714213af447aca4125342fd80721e, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:38,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/f57714213af447aca4125342fd80721e is 175, key is test_row_0/A:col10/1732765416866/Put/seqid=0 2024-11-28T03:43:39,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742496_1672 (size=31105) 2024-11-28T03:43:39,013 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/f57714213af447aca4125342fd80721e 2024-11-28T03:43:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:39,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:39,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/9c955be3952040b28be64437faa42f04 is 50, key is test_row_0/B:col10/1732765416866/Put/seqid=0 2024-11-28T03:43:39,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742497_1673 (size=12151) 2024-11-28T03:43:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765479069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:39,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765479071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765479187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:39,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765479188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:39,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765479397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:39,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765479397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:39,474 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/9c955be3952040b28be64437faa42f04 2024-11-28T03:43:39,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/12db82d518394d34a5b99ff0b5fdf61b is 50, key is test_row_0/C:col10/1732765416866/Put/seqid=0 2024-11-28T03:43:39,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742498_1674 (size=12151) 2024-11-28T03:43:39,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765479706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:39,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765479706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:39,890 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/12db82d518394d34a5b99ff0b5fdf61b 2024-11-28T03:43:39,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/f57714213af447aca4125342fd80721e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/f57714213af447aca4125342fd80721e 2024-11-28T03:43:39,898 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/f57714213af447aca4125342fd80721e, entries=150, sequenceid=197, filesize=30.4 K 2024-11-28T03:43:39,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/9c955be3952040b28be64437faa42f04 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/9c955be3952040b28be64437faa42f04 2024-11-28T03:43:39,902 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/9c955be3952040b28be64437faa42f04, entries=150, sequenceid=197, filesize=11.9 K 2024-11-28T03:43:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/12db82d518394d34a5b99ff0b5fdf61b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/12db82d518394d34a5b99ff0b5fdf61b 2024-11-28T03:43:39,906 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/12db82d518394d34a5b99ff0b5fdf61b, entries=150, sequenceid=197, filesize=11.9 K 2024-11-28T03:43:39,907 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6c2355f8fea173e178470cf1c753f473 in 1331ms, sequenceid=197, compaction requested=false 2024-11-28T03:43:39,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:39,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:39,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-28T03:43:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-28T03:43:39,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-28T03:43:39,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0970 sec 2024-11-28T03:43:39,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 2.1040 sec 2024-11-28T03:43:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T03:43:39,957 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:39,957 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2024-11-28T03:43:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T03:43:39,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:39,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:39,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T03:43:40,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-28T03:43:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:40,112 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:43:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:40,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283048a7582ace4b7fb12ceb9b99c268be_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765419068/Put/seqid=0 2024-11-28T03:43:40,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742499_1675 (size=12304) 2024-11-28T03:43:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:40,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:40,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765480267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:40,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765480271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T03:43:40,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765480376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:40,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765480386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:40,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:40,530 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283048a7582ace4b7fb12ceb9b99c268be_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283048a7582ace4b7fb12ceb9b99c268be_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:40,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/15f4eaeb9c66425bb4b33e1cc9c60abc, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:40,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/15f4eaeb9c66425bb4b33e1cc9c60abc is 175, key is test_row_0/A:col10/1732765419068/Put/seqid=0 2024-11-28T03:43:40,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742500_1676 (size=31105) 2024-11-28T03:43:40,538 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/15f4eaeb9c66425bb4b33e1cc9c60abc 2024-11-28T03:43:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/e453d52de77f4791a92b339e01413ff3 is 50, key is test_row_0/B:col10/1732765419068/Put/seqid=0 2024-11-28T03:43:40,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742501_1677 (size=12151) 2024-11-28T03:43:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T03:43:40,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765480586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:40,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765480596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:40,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765480896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:40,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765480906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:40,958 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/e453d52de77f4791a92b339e01413ff3 2024-11-28T03:43:40,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/fe385b8835c9423c8e319b18cdad19c0 is 50, key is test_row_0/C:col10/1732765419068/Put/seqid=0 2024-11-28T03:43:40,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742502_1678 (size=12151) 2024-11-28T03:43:40,993 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/fe385b8835c9423c8e319b18cdad19c0 2024-11-28T03:43:40,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/15f4eaeb9c66425bb4b33e1cc9c60abc as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/15f4eaeb9c66425bb4b33e1cc9c60abc 2024-11-28T03:43:41,002 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/15f4eaeb9c66425bb4b33e1cc9c60abc, entries=150, sequenceid=212, filesize=30.4 K 2024-11-28T03:43:41,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/e453d52de77f4791a92b339e01413ff3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/e453d52de77f4791a92b339e01413ff3 2024-11-28T03:43:41,006 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/e453d52de77f4791a92b339e01413ff3, entries=150, sequenceid=212, filesize=11.9 K 2024-11-28T03:43:41,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/fe385b8835c9423c8e319b18cdad19c0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/fe385b8835c9423c8e319b18cdad19c0 2024-11-28T03:43:41,010 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/fe385b8835c9423c8e319b18cdad19c0, entries=150, sequenceid=212, filesize=11.9 K 2024-11-28T03:43:41,011 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6c2355f8fea173e178470cf1c753f473 in 899ms, sequenceid=212, compaction requested=true 2024-11-28T03:43:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=179 2024-11-28T03:43:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=179 2024-11-28T03:43:41,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-28T03:43:41,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0530 sec 2024-11-28T03:43:41,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 1.0580 sec 2024-11-28T03:43:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T03:43:41,096 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:41,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees 2024-11-28T03:43:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-28T03:43:41,098 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:41,099 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:41,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-28T03:43:41,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-28T03:43:41,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:41,252 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:43:41,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:41,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:41,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:41,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:41,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:41,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:41,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128dea934eed2e7467aac6cc55f53acab70_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765420270/Put/seqid=0 2024-11-28T03:43:41,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742503_1679 (size=12304) 2024-11-28T03:43:41,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:41,267 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128dea934eed2e7467aac6cc55f53acab70_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dea934eed2e7467aac6cc55f53acab70_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:41,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/38dcf7ebda26432ca0ab9ccc5c881239, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:41,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/38dcf7ebda26432ca0ab9ccc5c881239 is 175, key is test_row_0/A:col10/1732765420270/Put/seqid=0 2024-11-28T03:43:41,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742504_1680 (size=31105) 2024-11-28T03:43:41,275 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/38dcf7ebda26432ca0ab9ccc5c881239 2024-11-28T03:43:41,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/af83d7c1269f445b946c3e069cbe6793 is 50, key is test_row_0/B:col10/1732765420270/Put/seqid=0 2024-11-28T03:43:41,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742505_1681 (size=12151) 2024-11-28T03:43:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:41,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:41,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765481384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765481387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765481388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765481407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-28T03:43:41,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765481416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765481496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765481496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765481496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,687 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/af83d7c1269f445b946c3e069cbe6793 2024-11-28T03:43:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/c3d980c346a74c8eaf0e5a68d30daa7e is 50, key is test_row_0/C:col10/1732765420270/Put/seqid=0 2024-11-28T03:43:41,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742506_1682 (size=12151) 2024-11-28T03:43:41,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765481707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765481707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:41,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765481707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:41,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:41,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-28T03:43:42,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765482016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765482017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765482017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,098 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/c3d980c346a74c8eaf0e5a68d30daa7e 2024-11-28T03:43:42,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/38dcf7ebda26432ca0ab9ccc5c881239 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/38dcf7ebda26432ca0ab9ccc5c881239 2024-11-28T03:43:42,105 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/38dcf7ebda26432ca0ab9ccc5c881239, entries=150, sequenceid=230, filesize=30.4 K 2024-11-28T03:43:42,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/af83d7c1269f445b946c3e069cbe6793 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/af83d7c1269f445b946c3e069cbe6793 2024-11-28T03:43:42,109 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/af83d7c1269f445b946c3e069cbe6793, entries=150, sequenceid=230, filesize=11.9 K 2024-11-28T03:43:42,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/c3d980c346a74c8eaf0e5a68d30daa7e as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/c3d980c346a74c8eaf0e5a68d30daa7e 2024-11-28T03:43:42,112 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/c3d980c346a74c8eaf0e5a68d30daa7e, entries=150, sequenceid=230, filesize=11.9 K 2024-11-28T03:43:42,113 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6c2355f8fea173e178470cf1c753f473 in 861ms, sequenceid=230, compaction requested=true 2024-11-28T03:43:42,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:42,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:42,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=181 2024-11-28T03:43:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=181 2024-11-28T03:43:42,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-28T03:43:42,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-11-28T03:43:42,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees in 1.0200 sec 2024-11-28T03:43:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-28T03:43:42,237 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:42,237 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=182, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees 2024-11-28T03:43:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-28T03:43:42,239 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=182, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:42,239 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=182, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:42,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-28T03:43:42,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-11-28T03:43:42,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:42,392 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:43:42,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:42,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:42,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:42,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:42,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:42,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:42,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ce9deb03f0094b7488d5af370b3943f9_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765421380/Put/seqid=0 2024-11-28T03:43:42,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742507_1683 (size=12304) 2024-11-28T03:43:42,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:42,406 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ce9deb03f0094b7488d5af370b3943f9_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ce9deb03f0094b7488d5af370b3943f9_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/ab82e51d82ce4e1b8546942d1c79f4e5, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/ab82e51d82ce4e1b8546942d1c79f4e5 is 175, key is test_row_0/A:col10/1732765421380/Put/seqid=0 2024-11-28T03:43:42,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742508_1684 (size=31105) 2024-11-28T03:43:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:42,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:42,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765482463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765482468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765482526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765482526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765482526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-28T03:43:42,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765482577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765482577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765482786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765482786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:42,811 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/ab82e51d82ce4e1b8546942d1c79f4e5 2024-11-28T03:43:42,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/f68d4d85d1924a0db32de0bf4130217b is 50, key is test_row_0/B:col10/1732765421380/Put/seqid=0 2024-11-28T03:43:42,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742509_1685 (size=12151) 2024-11-28T03:43:42,820 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/f68d4d85d1924a0db32de0bf4130217b 2024-11-28T03:43:42,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/2c433904bf1e4194a7cf05988254f325 is 50, key is test_row_0/C:col10/1732765421380/Put/seqid=0 2024-11-28T03:43:42,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742510_1686 (size=12151) 2024-11-28T03:43:42,828 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/2c433904bf1e4194a7cf05988254f325 2024-11-28T03:43:42,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/ab82e51d82ce4e1b8546942d1c79f4e5 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/ab82e51d82ce4e1b8546942d1c79f4e5 2024-11-28T03:43:42,834 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/ab82e51d82ce4e1b8546942d1c79f4e5, entries=150, sequenceid=244, filesize=30.4 K 2024-11-28T03:43:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/f68d4d85d1924a0db32de0bf4130217b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/f68d4d85d1924a0db32de0bf4130217b 2024-11-28T03:43:42,838 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/f68d4d85d1924a0db32de0bf4130217b, entries=150, sequenceid=244, filesize=11.9 K 2024-11-28T03:43:42,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/2c433904bf1e4194a7cf05988254f325 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/2c433904bf1e4194a7cf05988254f325 2024-11-28T03:43:42,842 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/2c433904bf1e4194a7cf05988254f325, entries=150, sequenceid=244, filesize=11.9 K 2024-11-28T03:43:42,842 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6c2355f8fea173e178470cf1c753f473 in 450ms, sequenceid=244, compaction requested=true 2024-11-28T03:43:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=183 2024-11-28T03:43:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=183 2024-11-28T03:43:42,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=183, resume processing ppid=182 2024-11-28T03:43:42,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, ppid=182, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 604 msec 2024-11-28T03:43:42,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees in 609 msec 2024-11-28T03:43:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-28T03:43:42,866 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:42,867 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=184, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees 2024-11-28T03:43:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-28T03:43:42,869 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=184, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:42,870 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=184, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:42,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=185, ppid=184, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-28T03:43:43,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-28T03:43:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:43,023 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T03:43:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112819a33932109741feb5a98c2ee683049e_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765422462/Put/seqid=0 2024-11-28T03:43:43,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742511_1687 (size=12404) 2024-11-28T03:43:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:43,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:43,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765483126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765483131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-28T03:43:43,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765483236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765483247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:43,438 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112819a33932109741feb5a98c2ee683049e_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112819a33932109741feb5a98c2ee683049e_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:43,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/8afd635982424dfeb3aa4cc1a70c34e6, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:43,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/8afd635982424dfeb3aa4cc1a70c34e6 is 175, key is test_row_0/A:col10/1732765422462/Put/seqid=0 2024-11-28T03:43:43,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742512_1688 (size=31205) 2024-11-28T03:43:43,444 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/8afd635982424dfeb3aa4cc1a70c34e6 2024-11-28T03:43:43,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765483446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/b445781235be4484b9f5c2318a1f41d0 is 50, key is test_row_0/B:col10/1732765422462/Put/seqid=0 2024-11-28T03:43:43,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742513_1689 (size=12251) 2024-11-28T03:43:43,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765483457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-28T03:43:43,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765483546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765483547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765483547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,550 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:43,551 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_1' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,551 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765483756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765483767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:43,855 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/b445781235be4484b9f5c2318a1f41d0 2024-11-28T03:43:43,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/51349bcb5d77443ebf2f9551adc6292b is 50, key is test_row_0/C:col10/1732765422462/Put/seqid=0 2024-11-28T03:43:43,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742514_1690 (size=12251) 2024-11-28T03:43:43,866 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/51349bcb5d77443ebf2f9551adc6292b 2024-11-28T03:43:43,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/8afd635982424dfeb3aa4cc1a70c34e6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8afd635982424dfeb3aa4cc1a70c34e6 2024-11-28T03:43:43,878 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8afd635982424dfeb3aa4cc1a70c34e6, entries=150, sequenceid=262, filesize=30.5 K 2024-11-28T03:43:43,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/b445781235be4484b9f5c2318a1f41d0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/b445781235be4484b9f5c2318a1f41d0 2024-11-28T03:43:43,883 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/b445781235be4484b9f5c2318a1f41d0, entries=150, sequenceid=262, filesize=12.0 K 2024-11-28T03:43:43,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/51349bcb5d77443ebf2f9551adc6292b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/51349bcb5d77443ebf2f9551adc6292b 2024-11-28T03:43:43,889 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/51349bcb5d77443ebf2f9551adc6292b, entries=150, sequenceid=262, filesize=12.0 K 2024-11-28T03:43:43,889 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6c2355f8fea173e178470cf1c753f473 in 867ms, sequenceid=262, compaction requested=true 2024-11-28T03:43:43,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:43,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:43,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=185 2024-11-28T03:43:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=185 2024-11-28T03:43:43,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=185, resume processing ppid=184 2024-11-28T03:43:43,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=185, ppid=184, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0210 sec 2024-11-28T03:43:43,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees in 1.0270 sec 2024-11-28T03:43:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-28T03:43:44,007 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:44,007 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=186, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees 2024-11-28T03:43:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-28T03:43:44,009 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=186, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:44,010 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=186, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:44,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=187, ppid=186, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-28T03:43:44,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2024-11-28T03:43:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:44,163 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:43:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:44,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112812e0cd57326c45558e3090eefa1bc363_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765423129/Put/seqid=0 2024-11-28T03:43:44,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742515_1691 (size=12454) 2024-11-28T03:43:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:44,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-28T03:43:44,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765484329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:44,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765484333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:44,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765484436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:44,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765484446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:44,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:44,578 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112812e0cd57326c45558e3090eefa1bc363_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112812e0cd57326c45558e3090eefa1bc363_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:44,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/0854e8a6d38d4fc28e1a5d02ae6276ca, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:44,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/0854e8a6d38d4fc28e1a5d02ae6276ca is 175, key is test_row_0/A:col10/1732765423129/Put/seqid=0 2024-11-28T03:43:44,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742516_1692 (size=31255) 2024-11-28T03:43:44,583 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/0854e8a6d38d4fc28e1a5d02ae6276ca 2024-11-28T03:43:44,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/59e1011c3da14fad92fbc031d9547ca3 is 50, key is test_row_0/B:col10/1732765423129/Put/seqid=0 2024-11-28T03:43:44,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742517_1693 (size=12301) 2024-11-28T03:43:44,602 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/59e1011c3da14fad92fbc031d9547ca3 2024-11-28T03:43:44,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/5e58fad7deb147e293c704195f9385a9 is 50, key is test_row_0/C:col10/1732765423129/Put/seqid=0 2024-11-28T03:43:44,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742518_1694 (size=12301) 2024-11-28T03:43:44,619 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/5e58fad7deb147e293c704195f9385a9 2024-11-28T03:43:44,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/0854e8a6d38d4fc28e1a5d02ae6276ca as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/0854e8a6d38d4fc28e1a5d02ae6276ca 2024-11-28T03:43:44,629 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/0854e8a6d38d4fc28e1a5d02ae6276ca, entries=150, sequenceid=276, filesize=30.5 K 2024-11-28T03:43:44,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/59e1011c3da14fad92fbc031d9547ca3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/59e1011c3da14fad92fbc031d9547ca3 2024-11-28T03:43:44,633 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/59e1011c3da14fad92fbc031d9547ca3, entries=150, sequenceid=276, filesize=12.0 K 2024-11-28T03:43:44,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/5e58fad7deb147e293c704195f9385a9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5e58fad7deb147e293c704195f9385a9 2024-11-28T03:43:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-28T03:43:44,637 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5e58fad7deb147e293c704195f9385a9, entries=150, sequenceid=276, filesize=12.0 K 2024-11-28T03:43:44,638 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6c2355f8fea173e178470cf1c753f473 in 475ms, sequenceid=276, compaction requested=true 2024-11-28T03:43:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=187 2024-11-28T03:43:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=187 2024-11-28T03:43:44,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=187, resume processing ppid=186 2024-11-28T03:43:44,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=187, ppid=186, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 629 msec 2024-11-28T03:43:44,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=186, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees in 635 msec 2024-11-28T03:43:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:44,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:43:44,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:44,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:44,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:44,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:44,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:44,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:44,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e24b6917b8064f9eba683443e51ee9f7_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765424649/Put/seqid=0 2024-11-28T03:43:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742519_1695 (size=14994) 2024-11-28T03:43:44,665 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:44,669 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e24b6917b8064f9eba683443e51ee9f7_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e24b6917b8064f9eba683443e51ee9f7_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:44,670 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/271dba4e56014b9a8eeb3c7838e4095b, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:44,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/271dba4e56014b9a8eeb3c7838e4095b is 175, key is test_row_0/A:col10/1732765424649/Put/seqid=0 2024-11-28T03:43:44,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742520_1696 (size=39949) 2024-11-28T03:43:44,683 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/271dba4e56014b9a8eeb3c7838e4095b 2024-11-28T03:43:44,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/a4a68d83f1ee42839195660e9c62d980 is 50, key is test_row_0/B:col10/1732765424649/Put/seqid=0 2024-11-28T03:43:44,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765484687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:44,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765484695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:44,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742521_1697 (size=12301) 2024-11-28T03:43:44,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765484806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:44,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765484807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765485016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765485016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/a4a68d83f1ee42839195660e9c62d980 2024-11-28T03:43:45,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/3885cbeac40944528966425104c0f2b3 is 50, key is test_row_0/C:col10/1732765424649/Put/seqid=0 2024-11-28T03:43:45,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742522_1698 (size=12301) 2024-11-28T03:43:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-28T03:43:45,147 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:45,147 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T03:43:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=188, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees 2024-11-28T03:43:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=188 2024-11-28T03:43:45,149 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=188, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T03:43:45,149 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=188, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T03:43:45,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=189, ppid=188, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T03:43:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=188 2024-11-28T03:43:45,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765485326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765485326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=188 2024-11-28T03:43:45,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/3885cbeac40944528966425104c0f2b3 2024-11-28T03:43:45,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/271dba4e56014b9a8eeb3c7838e4095b as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/271dba4e56014b9a8eeb3c7838e4095b 2024-11-28T03:43:45,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/271dba4e56014b9a8eeb3c7838e4095b, entries=200, sequenceid=295, filesize=39.0 K 2024-11-28T03:43:45,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/a4a68d83f1ee42839195660e9c62d980 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/a4a68d83f1ee42839195660e9c62d980 2024-11-28T03:43:45,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/a4a68d83f1ee42839195660e9c62d980, entries=150, sequenceid=295, filesize=12.0 K 2024-11-28T03:43:45,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/3885cbeac40944528966425104c0f2b3 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3885cbeac40944528966425104c0f2b3 2024-11-28T03:43:45,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3885cbeac40944528966425104c0f2b3, entries=150, sequenceid=295, filesize=12.0 K 2024-11-28T03:43:45,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6c2355f8fea173e178470cf1c753f473 in 879ms, sequenceid=295, compaction requested=true 2024-11-28T03:43:45,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:45,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T03:43:45,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:45,529 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-28T03:43:45,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T03:43:45,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:45,529 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-28T03:43:45,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c2355f8fea173e178470cf1c753f473:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T03:43:45,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:45,531 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 98018 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-28T03:43:45,531 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/B is initiating minor compaction (all files) 2024-11-28T03:43:45,531 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/B in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,531 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/bd3ffa67f9be4f0bb6caa01ad8790c75, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/9c955be3952040b28be64437faa42f04, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/e453d52de77f4791a92b339e01413ff3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/af83d7c1269f445b946c3e069cbe6793, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/f68d4d85d1924a0db32de0bf4130217b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/b445781235be4484b9f5c2318a1f41d0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/59e1011c3da14fad92fbc031d9547ca3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/a4a68d83f1ee42839195660e9c62d980] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=95.7 K 2024-11-28T03:43:45,531 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 258344 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-28T03:43:45,532 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/A is initiating minor compaction (all files) 2024-11-28T03:43:45,532 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/A in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,532 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/fa129e3df3c84a4bb72446eb2e3e193f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/f57714213af447aca4125342fd80721e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/15f4eaeb9c66425bb4b33e1cc9c60abc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/38dcf7ebda26432ca0ab9ccc5c881239, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/ab82e51d82ce4e1b8546942d1c79f4e5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8afd635982424dfeb3aa4cc1a70c34e6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/0854e8a6d38d4fc28e1a5d02ae6276ca, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/271dba4e56014b9a8eeb3c7838e4095b] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=252.3 K 2024-11-28T03:43:45,532 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=8 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,532 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. files: [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/fa129e3df3c84a4bb72446eb2e3e193f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/f57714213af447aca4125342fd80721e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/15f4eaeb9c66425bb4b33e1cc9c60abc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/38dcf7ebda26432ca0ab9ccc5c881239, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/ab82e51d82ce4e1b8546942d1c79f4e5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8afd635982424dfeb3aa4cc1a70c34e6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/0854e8a6d38d4fc28e1a5d02ae6276ca, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/271dba4e56014b9a8eeb3c7838e4095b] 2024-11-28T03:43:45,532 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3ffa67f9be4f0bb6caa01ad8790c75, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732765414663 2024-11-28T03:43:45,532 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa129e3df3c84a4bb72446eb2e3e193f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732765414663 2024-11-28T03:43:45,532 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c955be3952040b28be64437faa42f04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732765416854 2024-11-28T03:43:45,533 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting e453d52de77f4791a92b339e01413ff3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732765419043 2024-11-28T03:43:45,533 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f57714213af447aca4125342fd80721e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732765416854 2024-11-28T03:43:45,533 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting af83d7c1269f445b946c3e069cbe6793, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732765420255 2024-11-28T03:43:45,533 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15f4eaeb9c66425bb4b33e1cc9c60abc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732765419043 2024-11-28T03:43:45,533 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting f68d4d85d1924a0db32de0bf4130217b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732765421380 2024-11-28T03:43:45,533 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38dcf7ebda26432ca0ab9ccc5c881239, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732765420255 2024-11-28T03:43:45,533 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting b445781235be4484b9f5c2318a1f41d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732765422449 2024-11-28T03:43:45,533 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab82e51d82ce4e1b8546942d1c79f4e5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732765421380 2024-11-28T03:43:45,534 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 59e1011c3da14fad92fbc031d9547ca3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732765423117 2024-11-28T03:43:45,534 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8afd635982424dfeb3aa4cc1a70c34e6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732765422449 2024-11-28T03:43:45,534 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a68d83f1ee42839195660e9c62d980, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732765424301 2024-11-28T03:43:45,534 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0854e8a6d38d4fc28e1a5d02ae6276ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732765423117 2024-11-28T03:43:45,534 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 271dba4e56014b9a8eeb3c7838e4095b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732765424301 2024-11-28T03:43:45,552 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#B#compaction#597 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:45,553 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:45,554 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/728dab5ede7848c2b9a357538d303c52 is 50, key is test_row_0/B:col10/1732765424649/Put/seqid=0 2024-11-28T03:43:45,554 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128609b3579567045a38440d74471119116_6c2355f8fea173e178470cf1c753f473 store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:45,559 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128609b3579567045a38440d74471119116_6c2355f8fea173e178470cf1c753f473, store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:45,560 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128609b3579567045a38440d74471119116_6c2355f8fea173e178470cf1c753f473 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(8855): Flush requested on 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:45,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T03:43:45,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:45,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:45,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:45,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:45,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:45,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:45,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742523_1699 (size=4469) 2024-11-28T03:43:45,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742524_1700 (size=12983) 2024-11-28T03:43:45,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281d52de67cbca4ed6a79795beea29ed5d_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765424693/Put/seqid=0 2024-11-28T03:43:45,597 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/728dab5ede7848c2b9a357538d303c52 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/728dab5ede7848c2b9a357538d303c52 2024-11-28T03:43:45,601 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/B of 6c2355f8fea173e178470cf1c753f473 into 728dab5ede7848c2b9a357538d303c52(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:45,601 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:45,601 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/B, priority=8, startTime=1732765425529; duration=0sec 2024-11-28T03:43:45,602 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T03:43:45,602 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:B 2024-11-28T03:43:45,602 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-28T03:43:45,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742525_1701 (size=14994) 2024-11-28T03:43:45,604 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 98018 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-28T03:43:45,604 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1541): 6c2355f8fea173e178470cf1c753f473/C is initiating minor compaction (all files) 2024-11-28T03:43:45,604 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c2355f8fea173e178470cf1c753f473/C in TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,604 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/9f2f284ef1fd4df990d281a3d68e343a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/12db82d518394d34a5b99ff0b5fdf61b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/fe385b8835c9423c8e319b18cdad19c0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/c3d980c346a74c8eaf0e5a68d30daa7e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/2c433904bf1e4194a7cf05988254f325, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/51349bcb5d77443ebf2f9551adc6292b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5e58fad7deb147e293c704195f9385a9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3885cbeac40944528966425104c0f2b3] into tmpdir=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp, totalSize=95.7 K 2024-11-28T03:43:45,605 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f2f284ef1fd4df990d281a3d68e343a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732765414663 2024-11-28T03:43:45,605 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 12db82d518394d34a5b99ff0b5fdf61b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732765416854 2024-11-28T03:43:45,605 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting fe385b8835c9423c8e319b18cdad19c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732765419043 2024-11-28T03:43:45,606 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting c3d980c346a74c8eaf0e5a68d30daa7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732765420255 2024-11-28T03:43:45,606 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c433904bf1e4194a7cf05988254f325, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732765421380 2024-11-28T03:43:45,606 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 51349bcb5d77443ebf2f9551adc6292b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732765422449 2024-11-28T03:43:45,606 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e58fad7deb147e293c704195f9385a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732765423117 2024-11-28T03:43:45,606 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3885cbeac40944528966425104c0f2b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732765424301 2024-11-28T03:43:45,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:45,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:45,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,620 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#C#compaction#600 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:45,621 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/6d35e3be87104983aca204616791d2a4 is 50, key is test_row_0/C:col10/1732765424649/Put/seqid=0 2024-11-28T03:43:45,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765485613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765485622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765485622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742526_1702 (size=12983) 2024-11-28T03:43:45,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765485727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765485736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765485736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:45,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:45,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=188 2024-11-28T03:43:45,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765485836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765485847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:45,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:45,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:45,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765485936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765485947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765485947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:45,979 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c2355f8fea173e178470cf1c753f473#A#compaction#598 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T03:43:45,979 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/fa5c7cce12c141fab62b784e8551a63a is 175, key is test_row_0/A:col10/1732765424649/Put/seqid=0 2024-11-28T03:43:45,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742527_1703 (size=31937) 2024-11-28T03:43:45,987 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/fa5c7cce12c141fab62b784e8551a63a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/fa5c7cce12c141fab62b784e8551a63a 2024-11-28T03:43:45,991 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/A of 6c2355f8fea173e178470cf1c753f473 into fa5c7cce12c141fab62b784e8551a63a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:45,991 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:45,991 INFO [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/A, priority=8, startTime=1732765425529; duration=0sec 2024-11-28T03:43:45,991 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:45,991 DEBUG [RS:0;3a546af21488:34671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:A 2024-11-28T03:43:46,002 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:46,005 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281d52de67cbca4ed6a79795beea29ed5d_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281d52de67cbca4ed6a79795beea29ed5d_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:46,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/8669a90d39384b27a7297c296fbf4dbc, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:46,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/8669a90d39384b27a7297c296fbf4dbc is 175, key is test_row_0/A:col10/1732765424693/Put/seqid=0 2024-11-28T03:43:46,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742528_1704 (size=39949) 2024-11-28T03:43:46,033 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/6d35e3be87104983aca204616791d2a4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/6d35e3be87104983aca204616791d2a4 2024-11-28T03:43:46,037 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 6c2355f8fea173e178470cf1c753f473/C of 6c2355f8fea173e178470cf1c753f473 into 6d35e3be87104983aca204616791d2a4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T03:43:46,037 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:46,037 INFO [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., storeName=6c2355f8fea173e178470cf1c753f473/C, priority=8, startTime=1732765425529; duration=0sec 2024-11-28T03:43:46,037 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T03:43:46,037 DEBUG [RS:0;3a546af21488:34671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c2355f8fea173e178470cf1c753f473:C 2024-11-28T03:43:46,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:46,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:46,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:46,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:46,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765486246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:46,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:46,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765486257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765486257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=188 2024-11-28T03:43:46,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/8669a90d39384b27a7297c296fbf4dbc 2024-11-28T03:43:46,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/c39d0413b15447aab9865eeda81c7f25 is 50, key is test_row_0/B:col10/1732765424693/Put/seqid=0 2024-11-28T03:43:46,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742529_1705 (size=12301) 2024-11-28T03:43:46,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/c39d0413b15447aab9865eeda81c7f25 2024-11-28T03:43:46,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/12a672db984c4a1b81e7d5721cef6d72 is 50, key is test_row_0/C:col10/1732765424693/Put/seqid=0 2024-11-28T03:43:46,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742530_1706 (size=12301) 2024-11-28T03:43:46,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:46,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:46,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38602 deadline: 1732765486758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:46,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38580 deadline: 1732765486766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38586 deadline: 1732765486767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:46,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/12a672db984c4a1b81e7d5721cef6d72 2024-11-28T03:43:46,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:46,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. as already flushing 2024-11-28T03:43:46,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(58): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4177): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:43:46,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/8669a90d39384b27a7297c296fbf4dbc as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8669a90d39384b27a7297c296fbf4dbc 2024-11-28T03:43:46,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8669a90d39384b27a7297c296fbf4dbc, entries=200, sequenceid=309, filesize=39.0 K 2024-11-28T03:43:46,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/c39d0413b15447aab9865eeda81c7f25 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/c39d0413b15447aab9865eeda81c7f25 2024-11-28T03:43:46,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/c39d0413b15447aab9865eeda81c7f25, entries=150, sequenceid=309, filesize=12.0 K 2024-11-28T03:43:46,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/12a672db984c4a1b81e7d5721cef6d72 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/12a672db984c4a1b81e7d5721cef6d72 2024-11-28T03:43:46,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38652 deadline: 1732765486856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:46,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T03:43:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1732765486857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 2024-11-28T03:43:46,858 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:46,858 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3a546af21488,34671,1732765249447 for 'test_row_2' in 6c2355f8fea173e178470cf1c753f473 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T03:43:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 , the old value is region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T03:43:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., hostname=3a546af21488,34671,1732765249447, seqNum=5 because the exception is null or not the one we care about 2024-11-28T03:43:46,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/12a672db984c4a1b81e7d5721cef6d72, entries=150, sequenceid=309, filesize=12.0 K 2024-11-28T03:43:46,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 6c2355f8fea173e178470cf1c753f473 in 1297ms, sequenceid=309, compaction requested=false 2024-11-28T03:43:46,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:46,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-11-28T03:43:46,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:46,988 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T03:43:46,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:46,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:46,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:46,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:46,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:46,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:46,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281f706609d8da4125bc982665087647c5_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765425610/Put/seqid=0 2024-11-28T03:43:47,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742531_1707 (size=12454) 2024-11-28T03:43:47,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,019 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281f706609d8da4125bc982665087647c5_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281f706609d8da4125bc982665087647c5_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:47,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/b7e554ef4ef149a795dd7ab392aea353, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:47,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/b7e554ef4ef149a795dd7ab392aea353 is 175, key is test_row_0/A:col10/1732765425610/Put/seqid=0 2024-11-28T03:43:47,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742532_1708 (size=31255) 2024-11-28T03:43:47,026 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/b7e554ef4ef149a795dd7ab392aea353 2024-11-28T03:43:47,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/f6b72cb466704b0bad491cde4d5cf535 is 50, key is test_row_0/B:col10/1732765425610/Put/seqid=0 2024-11-28T03:43:47,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742533_1709 (size=12301) 2024-11-28T03:43:47,038 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/f6b72cb466704b0bad491cde4d5cf535 2024-11-28T03:43:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/afca282dc84d4490a397bbd089471c4f is 50, key is test_row_0/C:col10/1732765425610/Put/seqid=0 2024-11-28T03:43:47,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742534_1710 (size=12301) 2024-11-28T03:43:47,054 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/afca282dc84d4490a397bbd089471c4f 2024-11-28T03:43:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/b7e554ef4ef149a795dd7ab392aea353 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b7e554ef4ef149a795dd7ab392aea353 2024-11-28T03:43:47,064 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b7e554ef4ef149a795dd7ab392aea353, entries=150, sequenceid=331, filesize=30.5 K 2024-11-28T03:43:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/f6b72cb466704b0bad491cde4d5cf535 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/f6b72cb466704b0bad491cde4d5cf535 2024-11-28T03:43:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,068 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/f6b72cb466704b0bad491cde4d5cf535, entries=150, sequenceid=331, filesize=12.0 K 2024-11-28T03:43:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/afca282dc84d4490a397bbd089471c4f as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/afca282dc84d4490a397bbd089471c4f 2024-11-28T03:43:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,075 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/afca282dc84d4490a397bbd089471c4f, entries=150, sequenceid=331, filesize=12.0 K 2024-11-28T03:43:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,076 INFO [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 6c2355f8fea173e178470cf1c753f473 in 88ms, sequenceid=331, compaction requested=true 2024-11-28T03:43:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2603): Flush status journal for 6c2355f8fea173e178470cf1c753f473: 2024-11-28T03:43:47,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:47,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=189 2024-11-28T03:43:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster(4169): Remote procedure done, pid=189 2024-11-28T03:43:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=189, resume processing ppid=188 2024-11-28T03:43:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=189, ppid=188, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9290 sec 2024-11-28T03:43:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=188, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees in 1.9340 sec 2024-11-28T03:43:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,209 INFO [Thread-2732 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2732. 2024-11-28T03:43:47,209 DEBUG [Thread-2732 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:47,209 DEBUG [Thread-2732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,209 DEBUG [Thread-2732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,212 INFO [Thread-2734 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2734. 2024-11-28T03:43:47,212 DEBUG [Thread-2734 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:47,212 DEBUG [Thread-2734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,212 DEBUG [Thread-2734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,214 INFO [Thread-2738 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2738. 2024-11-28T03:43:47,214 DEBUG [Thread-2738 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:47,214 DEBUG [Thread-2738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,214 DEBUG [Thread-2738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,233 INFO [Thread-2730 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2730. 2024-11-28T03:43:47,233 DEBUG [Thread-2730 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:47,233 DEBUG [Thread-2730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:47,234 DEBUG [Thread-2730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:47,236 INFO [Thread-2736 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2736. 2024-11-28T03:43:47,236 DEBUG [Thread-2736 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:47,236 DEBUG [Thread-2736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,236 DEBUG [Thread-2736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=188 2024-11-28T03:43:47,296 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:47,605 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T03:43:47,781 INFO [Thread-2721 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2721. 2024-11-28T03:43:47,781 DEBUG [Thread-2721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:47,781 DEBUG [Thread-2721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,781 DEBUG [Thread-2721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:47,789 INFO [Thread-2727 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2727. 2024-11-28T03:43:47,789 INFO [Thread-2725 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2725. 2024-11-28T03:43:47,789 DEBUG [Thread-2727 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:47,789 DEBUG [Thread-2725 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:47,789 DEBUG [Thread-2727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,789 DEBUG [Thread-2725 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,790 DEBUG [Thread-2725 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,790 DEBUG [Thread-2727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:47,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:47,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:48,868 INFO [Thread-2723 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2723. 2024-11-28T03:43:48,868 DEBUG [Thread-2723 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:48,868 DEBUG [Thread-2723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:48,868 DEBUG [Thread-2723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:48,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:48,888 INFO [Thread-2719 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2719. 2024-11-28T03:43:48,888 DEBUG [Thread-2719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T03:43:48,888 DEBUG [Thread-2719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:48,888 DEBUG [Thread-2719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:48,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2627 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2583 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2605 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2631 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2592 2024-11-28T03:43:48,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T03:43:48,888 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:43:48,888 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:43:48,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:48,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:48,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:48,889 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T03:43:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=190, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=190 2024-11-28T03:43:48,892 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765428891"}]},"ts":"1732765428891"} 2024-11-28T03:43:48,893 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T03:43:48,893 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T03:43:48,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=191, ppid=190, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T03:43:48,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=192, ppid=191, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, UNASSIGN}] 2024-11-28T03:43:48,895 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=192, ppid=191, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, UNASSIGN 2024-11-28T03:43:48,896 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=192 updating hbase:meta row=6c2355f8fea173e178470cf1c753f473, regionState=CLOSING, regionLocation=3a546af21488,34671,1732765249447 2024-11-28T03:43:48,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=192, ppid=191, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, UNASSIGN because future has completed 2024-11-28T03:43:48,897 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T03:43:48,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=193, ppid=192, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447}] 2024-11-28T03:43:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=190 2024-11-28T03:43:49,049 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] handler.UnassignRegionHandler(122): Close 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:49,049 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T03:43:49,049 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegion(1722): Closing 6c2355f8fea173e178470cf1c753f473, disabling compactions & flushes 2024-11-28T03:43:49,049 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:49,049 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:49,049 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. after waiting 0 ms 2024-11-28T03:43:49,049 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:49,049 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegion(2902): Flushing 6c2355f8fea173e178470cf1c753f473 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T03:43:49,050 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=A 2024-11-28T03:43:49,050 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:49,050 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=B 2024-11-28T03:43:49,050 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:49,050 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c2355f8fea173e178470cf1c753f473, store=C 2024-11-28T03:43:49,050 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T03:43:49,055 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128463a2f620c8b43f4bb3789746d292af2_6c2355f8fea173e178470cf1c753f473 is 50, key is test_row_0/A:col10/1732765427777/Put/seqid=0 2024-11-28T03:43:49,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742535_1711 (size=9914) 2024-11-28T03:43:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=190 2024-11-28T03:43:49,460 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:43:49,464 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128463a2f620c8b43f4bb3789746d292af2_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128463a2f620c8b43f4bb3789746d292af2_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:49,465 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/a21b09461f26484d824fb32f8f08f261, store: [table=TestAcidGuarantees family=A region=6c2355f8fea173e178470cf1c753f473] 2024-11-28T03:43:49,465 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/a21b09461f26484d824fb32f8f08f261 is 175, key is test_row_0/A:col10/1732765427777/Put/seqid=0 2024-11-28T03:43:49,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742536_1712 (size=22561) 2024-11-28T03:43:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=190 2024-11-28T03:43:49,869 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/a21b09461f26484d824fb32f8f08f261 2024-11-28T03:43:49,874 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/25e0a80167364a138577c8260754dd75 is 50, key is test_row_0/B:col10/1732765427777/Put/seqid=0 2024-11-28T03:43:49,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742537_1713 (size=9857) 2024-11-28T03:43:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=190 2024-11-28T03:43:50,277 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/25e0a80167364a138577c8260754dd75 2024-11-28T03:43:50,282 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/fa8365d8870749a193c02b97b8fc25c9 is 50, key is test_row_0/C:col10/1732765427777/Put/seqid=0 2024-11-28T03:43:50,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742538_1714 (size=9857) 2024-11-28T03:43:50,686 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/fa8365d8870749a193c02b97b8fc25c9 2024-11-28T03:43:50,689 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/A/a21b09461f26484d824fb32f8f08f261 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a21b09461f26484d824fb32f8f08f261 2024-11-28T03:43:50,692 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a21b09461f26484d824fb32f8f08f261, entries=100, sequenceid=339, filesize=22.0 K 2024-11-28T03:43:50,692 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/B/25e0a80167364a138577c8260754dd75 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/25e0a80167364a138577c8260754dd75 2024-11-28T03:43:50,695 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/25e0a80167364a138577c8260754dd75, entries=100, sequenceid=339, filesize=9.6 K 2024-11-28T03:43:50,696 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/.tmp/C/fa8365d8870749a193c02b97b8fc25c9 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/fa8365d8870749a193c02b97b8fc25c9 2024-11-28T03:43:50,699 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/fa8365d8870749a193c02b97b8fc25c9, entries=100, sequenceid=339, filesize=9.6 K 2024-11-28T03:43:50,699 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6c2355f8fea173e178470cf1c753f473 in 1650ms, sequenceid=339, compaction requested=true 2024-11-28T03:43:50,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/1066cb424aae4d328009f0591633de86, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a6c8ac006cfc4df48c20012a40db43e3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7414fc845f5b450e8d6a6b4b44fde4d7, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8b8637c3175b48b9ae033d48c26028df, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b122a1425b9f4ea7b5c07527b1875bbd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/e072b19124d84e1394a0e80de845f091, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4e442ed4c6514af99da517261269b4ff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/39c40a6d1a2144b089fcc25c0a18cf08, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/d9f98e686e654e46af88b07ac88c51e0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7a558c5fed6b4b058c6a5c9cdeb57580, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/fa129e3df3c84a4bb72446eb2e3e193f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/991304bc8b1d43249f76f319d27743b9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/f57714213af447aca4125342fd80721e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/15f4eaeb9c66425bb4b33e1cc9c60abc, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/38dcf7ebda26432ca0ab9ccc5c881239, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/ab82e51d82ce4e1b8546942d1c79f4e5, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8afd635982424dfeb3aa4cc1a70c34e6, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/0854e8a6d38d4fc28e1a5d02ae6276ca, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/271dba4e56014b9a8eeb3c7838e4095b] to archive 2024-11-28T03:43:50,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:43:50,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/1066cb424aae4d328009f0591633de86 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/1066cb424aae4d328009f0591633de86 2024-11-28T03:43:50,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a6c8ac006cfc4df48c20012a40db43e3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a6c8ac006cfc4df48c20012a40db43e3 2024-11-28T03:43:50,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7414fc845f5b450e8d6a6b4b44fde4d7 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7414fc845f5b450e8d6a6b4b44fde4d7 2024-11-28T03:43:50,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8b8637c3175b48b9ae033d48c26028df to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8b8637c3175b48b9ae033d48c26028df 2024-11-28T03:43:50,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4ba5a1cef8ce4b3dbc9fc2bd40c3734d 2024-11-28T03:43:50,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b122a1425b9f4ea7b5c07527b1875bbd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b122a1425b9f4ea7b5c07527b1875bbd 2024-11-28T03:43:50,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/e072b19124d84e1394a0e80de845f091 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/e072b19124d84e1394a0e80de845f091 2024-11-28T03:43:50,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4e442ed4c6514af99da517261269b4ff to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/4e442ed4c6514af99da517261269b4ff 2024-11-28T03:43:50,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/39c40a6d1a2144b089fcc25c0a18cf08 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/39c40a6d1a2144b089fcc25c0a18cf08 2024-11-28T03:43:50,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/d9f98e686e654e46af88b07ac88c51e0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/d9f98e686e654e46af88b07ac88c51e0 2024-11-28T03:43:50,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7a558c5fed6b4b058c6a5c9cdeb57580 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/7a558c5fed6b4b058c6a5c9cdeb57580 2024-11-28T03:43:50,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/fa129e3df3c84a4bb72446eb2e3e193f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/fa129e3df3c84a4bb72446eb2e3e193f 2024-11-28T03:43:50,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/991304bc8b1d43249f76f319d27743b9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/991304bc8b1d43249f76f319d27743b9 2024-11-28T03:43:50,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/f57714213af447aca4125342fd80721e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/f57714213af447aca4125342fd80721e 2024-11-28T03:43:50,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/15f4eaeb9c66425bb4b33e1cc9c60abc to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/15f4eaeb9c66425bb4b33e1cc9c60abc 2024-11-28T03:43:50,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/38dcf7ebda26432ca0ab9ccc5c881239 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/38dcf7ebda26432ca0ab9ccc5c881239 2024-11-28T03:43:50,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/ab82e51d82ce4e1b8546942d1c79f4e5 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/ab82e51d82ce4e1b8546942d1c79f4e5 2024-11-28T03:43:50,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8afd635982424dfeb3aa4cc1a70c34e6 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8afd635982424dfeb3aa4cc1a70c34e6 2024-11-28T03:43:50,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/0854e8a6d38d4fc28e1a5d02ae6276ca to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/0854e8a6d38d4fc28e1a5d02ae6276ca 2024-11-28T03:43:50,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/271dba4e56014b9a8eeb3c7838e4095b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/271dba4e56014b9a8eeb3c7838e4095b 2024-11-28T03:43:50,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/74733dae86094cd0be345a322168f040, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/900702700b634daa87f2e9e281b968ff, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/582811fc7cd842e8bd3b5a5d565a48ec, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/0a922a65f1a64c749f57f812c9506f2f, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/68a08d6b14704df8b39c34dfa8f4d00c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/90916ab6b0694e11bafb9a8dbdc291df, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/8278ffa10d2d4503930fbe22c776fe0d, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/61e115abf669425198991b37def619c0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/415a66a9c16f45dfb085a863053cfa6e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/2b9b0c16ec054f85ab2bb1f3fef144eb, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/7a1187aa1fa2459baa585565c9111cbb, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/bd3ffa67f9be4f0bb6caa01ad8790c75, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/71c7649ad2354b6c8881c92832eea5d8, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/9c955be3952040b28be64437faa42f04, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/e453d52de77f4791a92b339e01413ff3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/af83d7c1269f445b946c3e069cbe6793, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/f68d4d85d1924a0db32de0bf4130217b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/b445781235be4484b9f5c2318a1f41d0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/59e1011c3da14fad92fbc031d9547ca3, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/a4a68d83f1ee42839195660e9c62d980] to archive 2024-11-28T03:43:50,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:43:50,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/74733dae86094cd0be345a322168f040 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/74733dae86094cd0be345a322168f040 2024-11-28T03:43:50,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/900702700b634daa87f2e9e281b968ff to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/900702700b634daa87f2e9e281b968ff 2024-11-28T03:43:50,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/582811fc7cd842e8bd3b5a5d565a48ec to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/582811fc7cd842e8bd3b5a5d565a48ec 2024-11-28T03:43:50,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/0a922a65f1a64c749f57f812c9506f2f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/0a922a65f1a64c749f57f812c9506f2f 2024-11-28T03:43:50,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/68a08d6b14704df8b39c34dfa8f4d00c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/68a08d6b14704df8b39c34dfa8f4d00c 2024-11-28T03:43:50,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/90916ab6b0694e11bafb9a8dbdc291df to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/90916ab6b0694e11bafb9a8dbdc291df 2024-11-28T03:43:50,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/8278ffa10d2d4503930fbe22c776fe0d to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/8278ffa10d2d4503930fbe22c776fe0d 2024-11-28T03:43:50,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/61e115abf669425198991b37def619c0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/61e115abf669425198991b37def619c0 2024-11-28T03:43:50,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/415a66a9c16f45dfb085a863053cfa6e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/415a66a9c16f45dfb085a863053cfa6e 2024-11-28T03:43:50,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/2b9b0c16ec054f85ab2bb1f3fef144eb to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/2b9b0c16ec054f85ab2bb1f3fef144eb 2024-11-28T03:43:50,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/7a1187aa1fa2459baa585565c9111cbb to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/7a1187aa1fa2459baa585565c9111cbb 2024-11-28T03:43:50,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/bd3ffa67f9be4f0bb6caa01ad8790c75 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/bd3ffa67f9be4f0bb6caa01ad8790c75 2024-11-28T03:43:50,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/71c7649ad2354b6c8881c92832eea5d8 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/71c7649ad2354b6c8881c92832eea5d8 2024-11-28T03:43:50,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/9c955be3952040b28be64437faa42f04 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/9c955be3952040b28be64437faa42f04 2024-11-28T03:43:50,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/e453d52de77f4791a92b339e01413ff3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/e453d52de77f4791a92b339e01413ff3 2024-11-28T03:43:50,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/af83d7c1269f445b946c3e069cbe6793 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/af83d7c1269f445b946c3e069cbe6793 2024-11-28T03:43:50,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/f68d4d85d1924a0db32de0bf4130217b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/f68d4d85d1924a0db32de0bf4130217b 2024-11-28T03:43:50,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/b445781235be4484b9f5c2318a1f41d0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/b445781235be4484b9f5c2318a1f41d0 2024-11-28T03:43:50,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/59e1011c3da14fad92fbc031d9547ca3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/59e1011c3da14fad92fbc031d9547ca3 2024-11-28T03:43:50,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/a4a68d83f1ee42839195660e9c62d980 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/a4a68d83f1ee42839195660e9c62d980 2024-11-28T03:43:50,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/58e8eb27df11478f919f3e85425edd84, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/e99cf85302d045e3a7d49f32db7e3f37, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5f70d40a7e8e497788d6654919652a10, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/6af29c183e8741ac9eb71b447d7de6dd, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3b371558478848e3a3ee49de80713486, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/53230c6c622342769423b4ac0120a17a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/d638c4c535a84ee29c9965130510433a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/05b86844589a4bccb64bc8512452d141, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/920e9a294be64faa95ce64cff5b605e2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/111962b1c2854cb29948d9dc0b5b5a5c, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/27182b8363d44794bac808f09b82ae42, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/9f2f284ef1fd4df990d281a3d68e343a, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3c0400774b274225a1c0eccb99fa3fa2, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/12db82d518394d34a5b99ff0b5fdf61b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/fe385b8835c9423c8e319b18cdad19c0, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/c3d980c346a74c8eaf0e5a68d30daa7e, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/2c433904bf1e4194a7cf05988254f325, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/51349bcb5d77443ebf2f9551adc6292b, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5e58fad7deb147e293c704195f9385a9, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3885cbeac40944528966425104c0f2b3] to archive 2024-11-28T03:43:50,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T03:43:50,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/58e8eb27df11478f919f3e85425edd84 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/58e8eb27df11478f919f3e85425edd84 2024-11-28T03:43:50,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/e99cf85302d045e3a7d49f32db7e3f37 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/e99cf85302d045e3a7d49f32db7e3f37 2024-11-28T03:43:50,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5f70d40a7e8e497788d6654919652a10 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5f70d40a7e8e497788d6654919652a10 2024-11-28T03:43:50,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/6af29c183e8741ac9eb71b447d7de6dd to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/6af29c183e8741ac9eb71b447d7de6dd 2024-11-28T03:43:50,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3b371558478848e3a3ee49de80713486 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3b371558478848e3a3ee49de80713486 2024-11-28T03:43:50,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/53230c6c622342769423b4ac0120a17a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/53230c6c622342769423b4ac0120a17a 2024-11-28T03:43:50,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/d638c4c535a84ee29c9965130510433a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/d638c4c535a84ee29c9965130510433a 2024-11-28T03:43:50,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/05b86844589a4bccb64bc8512452d141 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/05b86844589a4bccb64bc8512452d141 2024-11-28T03:43:50,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/920e9a294be64faa95ce64cff5b605e2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/920e9a294be64faa95ce64cff5b605e2 2024-11-28T03:43:50,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/111962b1c2854cb29948d9dc0b5b5a5c to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/111962b1c2854cb29948d9dc0b5b5a5c 2024-11-28T03:43:50,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/27182b8363d44794bac808f09b82ae42 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/27182b8363d44794bac808f09b82ae42 2024-11-28T03:43:50,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/9f2f284ef1fd4df990d281a3d68e343a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/9f2f284ef1fd4df990d281a3d68e343a 2024-11-28T03:43:50,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3c0400774b274225a1c0eccb99fa3fa2 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3c0400774b274225a1c0eccb99fa3fa2 2024-11-28T03:43:50,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/12db82d518394d34a5b99ff0b5fdf61b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/12db82d518394d34a5b99ff0b5fdf61b 2024-11-28T03:43:50,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/fe385b8835c9423c8e319b18cdad19c0 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/fe385b8835c9423c8e319b18cdad19c0 2024-11-28T03:43:50,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/c3d980c346a74c8eaf0e5a68d30daa7e to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/c3d980c346a74c8eaf0e5a68d30daa7e 2024-11-28T03:43:50,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/2c433904bf1e4194a7cf05988254f325 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/2c433904bf1e4194a7cf05988254f325 2024-11-28T03:43:50,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/51349bcb5d77443ebf2f9551adc6292b to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/51349bcb5d77443ebf2f9551adc6292b 2024-11-28T03:43:50,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5e58fad7deb147e293c704195f9385a9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/5e58fad7deb147e293c704195f9385a9 2024-11-28T03:43:50,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3885cbeac40944528966425104c0f2b3 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/3885cbeac40944528966425104c0f2b3 2024-11-28T03:43:50,756 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/recovered.edits/342.seqid, newMaxSeqId=342, maxSeqId=4 2024-11-28T03:43:50,757 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. 2024-11-28T03:43:50,757 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] regionserver.HRegion(1676): Region close journal for 6c2355f8fea173e178470cf1c753f473: Waiting for close lock at 1732765429049Running coprocessor pre-close hooks at 1732765429049Disabling compacts and flushes for region at 1732765429049Disabling writes for close at 1732765429049Obtaining lock to block concurrent updates at 1732765429049Preparing flush snapshotting stores in 6c2355f8fea173e178470cf1c753f473 at 1732765429049Finished memstore snapshotting TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732765429050 (+1 ms)Flushing stores of TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473. at 1732765429050Flushing 6c2355f8fea173e178470cf1c753f473/A: creating writer at 1732765429050Flushing mob file 6c2355f8fea173e178470cf1c753f473/A: appending metadata at 1732765429055 (+5 ms)Flushing mob file 6c2355f8fea173e178470cf1c753f473/A: closing flushed file at 1732765429055Flushing 6c2355f8fea173e178470cf1c753f473/A: appending metadata at 1732765429465 (+410 ms)Flushing 6c2355f8fea173e178470cf1c753f473/A: closing flushed file at 1732765429465Flushing 6c2355f8fea173e178470cf1c753f473/B: creating writer at 1732765429872 (+407 ms)Flushing 6c2355f8fea173e178470cf1c753f473/B: appending metadata at 1732765429874 (+2 ms)Flushing 6c2355f8fea173e178470cf1c753f473/B: closing flushed file at 1732765429874Flushing 6c2355f8fea173e178470cf1c753f473/C: creating writer at 1732765430280 (+406 ms)Flushing 6c2355f8fea173e178470cf1c753f473/C: appending metadata at 1732765430282 (+2 ms)Flushing 6c2355f8fea173e178470cf1c753f473/C: closing flushed file at 1732765430282Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25e69a8f: reopening flushed file at 1732765430689 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6474dcdc: reopening flushed file at 1732765430692 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@576081b6: reopening flushed file at 1732765430695 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6c2355f8fea173e178470cf1c753f473 in 1650ms, sequenceid=339, compaction requested=true at 1732765430699 (+4 ms)Writing region close event to WAL at 1732765430754 (+55 ms)Running coprocessor post-close hooks at 1732765430757 (+3 ms)Closed at 1732765430757 2024-11-28T03:43:50,758 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION, pid=193}] handler.UnassignRegionHandler(157): Closed 6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:50,759 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=192 updating hbase:meta row=6c2355f8fea173e178470cf1c753f473, regionState=CLOSED 2024-11-28T03:43:50,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=193, ppid=192, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 because future has completed 2024-11-28T03:43:50,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=193, resume processing ppid=192 2024-11-28T03:43:50,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=193, ppid=192, state=SUCCESS, hasLock=false; CloseRegionProcedure 6c2355f8fea173e178470cf1c753f473, server=3a546af21488,34671,1732765249447 in 1.8630 sec 2024-11-28T03:43:50,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=192, resume processing ppid=191 2024-11-28T03:43:50,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=192, ppid=191, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c2355f8fea173e178470cf1c753f473, UNASSIGN in 1.8670 sec 2024-11-28T03:43:50,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=191, resume processing ppid=190 2024-11-28T03:43:50,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=191, ppid=190, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8710 sec 2024-11-28T03:43:50,766 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732765430766"}]},"ts":"1732765430766"} 2024-11-28T03:43:50,767 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T03:43:50,767 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T03:43:50,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=190, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8790 sec 2024-11-28T03:43:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=190 2024-11-28T03:43:51,036 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:51,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T03:43:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] procedure2.ProcedureExecutor(1139): Stored pid=194, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:51,038 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=194, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=194 2024-11-28T03:43:51,039 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=194, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:51,040 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,042 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C, FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/recovered.edits] 2024-11-28T03:43:51,044 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8669a90d39384b27a7297c296fbf4dbc to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/8669a90d39384b27a7297c296fbf4dbc 2024-11-28T03:43:51,045 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a21b09461f26484d824fb32f8f08f261 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/a21b09461f26484d824fb32f8f08f261 2024-11-28T03:43:51,045 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b7e554ef4ef149a795dd7ab392aea353 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/b7e554ef4ef149a795dd7ab392aea353 2024-11-28T03:43:51,046 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/fa5c7cce12c141fab62b784e8551a63a to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/A/fa5c7cce12c141fab62b784e8551a63a 2024-11-28T03:43:51,048 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/25e0a80167364a138577c8260754dd75 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/25e0a80167364a138577c8260754dd75 2024-11-28T03:43:51,049 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/728dab5ede7848c2b9a357538d303c52 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/728dab5ede7848c2b9a357538d303c52 2024-11-28T03:43:51,050 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/c39d0413b15447aab9865eeda81c7f25 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/c39d0413b15447aab9865eeda81c7f25 2024-11-28T03:43:51,050 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/f6b72cb466704b0bad491cde4d5cf535 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/B/f6b72cb466704b0bad491cde4d5cf535 2024-11-28T03:43:51,052 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/12a672db984c4a1b81e7d5721cef6d72 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/12a672db984c4a1b81e7d5721cef6d72 2024-11-28T03:43:51,053 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/6d35e3be87104983aca204616791d2a4 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/6d35e3be87104983aca204616791d2a4 2024-11-28T03:43:51,053 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/afca282dc84d4490a397bbd089471c4f to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/afca282dc84d4490a397bbd089471c4f 2024-11-28T03:43:51,054 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/fa8365d8870749a193c02b97b8fc25c9 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/C/fa8365d8870749a193c02b97b8fc25c9 2024-11-28T03:43:51,056 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/recovered.edits/342.seqid to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473/recovered.edits/342.seqid 2024-11-28T03:43:51,057 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/default/TestAcidGuarantees/6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,057 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T03:43:51,057 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T03:43:51,058 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T03:43:51,060 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112812e0cd57326c45558e3090eefa1bc363_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112812e0cd57326c45558e3090eefa1bc363_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,061 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112819a33932109741feb5a98c2ee683049e_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112819a33932109741feb5a98c2ee683049e_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,062 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281d52de67cbca4ed6a79795beea29ed5d_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281d52de67cbca4ed6a79795beea29ed5d_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,063 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281f706609d8da4125bc982665087647c5_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281f706609d8da4125bc982665087647c5_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,064 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283048a7582ace4b7fb12ceb9b99c268be_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283048a7582ace4b7fb12ceb9b99c268be_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,064 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128463a2f620c8b43f4bb3789746d292af2_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128463a2f620c8b43f4bb3789746d292af2_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,065 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112852b751b646cd4ccfb57067a64c75d92a_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112852b751b646cd4ccfb57067a64c75d92a_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,066 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112859d3172e3a224083807f83249e0bf681_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112859d3172e3a224083807f83249e0bf681_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,067 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285ca6542cd4ea44de89eb940a4adf6d4b_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285ca6542cd4ea44de89eb940a4adf6d4b_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,068 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128603e7e3e09d74751a597e514fcfb0484_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128603e7e3e09d74751a597e514fcfb0484_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,069 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128724f7ba354ec4ae8b64a2fb131891346_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128724f7ba354ec4ae8b64a2fb131891346_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,070 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287cdef6c16e61401c933107f56bd03990_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287cdef6c16e61401c933107f56bd03990_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,070 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112881b62f39d8d7490f9f95427ba21b0392_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112881b62f39d8d7490f9f95427ba21b0392_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,071 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ce9deb03f0094b7488d5af370b3943f9_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ce9deb03f0094b7488d5af370b3943f9_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,072 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d52cb557c30f4146bc789cede5296f7a_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d52cb557c30f4146bc789cede5296f7a_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,073 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dea934eed2e7467aac6cc55f53acab70_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dea934eed2e7467aac6cc55f53acab70_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,074 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e24b6917b8064f9eba683443e51ee9f7_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e24b6917b8064f9eba683443e51ee9f7_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,074 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e90c2e8b1b1c42298f9ed4506c78e974_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e90c2e8b1b1c42298f9ed4506c78e974_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,075 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ec7cbf15fc454bc0824ea0d1669c7d2f_6c2355f8fea173e178470cf1c753f473 to hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ec7cbf15fc454bc0824ea0d1669c7d2f_6c2355f8fea173e178470cf1c753f473 2024-11-28T03:43:51,076 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T03:43:51,078 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=194, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:51,079 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T03:43:51,081 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T03:43:51,082 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=194, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:51,082 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T03:43:51,082 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732765431082"}]},"ts":"9223372036854775807"} 2024-11-28T03:43:51,083 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T03:43:51,083 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6c2355f8fea173e178470cf1c753f473, NAME => 'TestAcidGuarantees,,1732765404085.6c2355f8fea173e178470cf1c753f473.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T03:43:51,083 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T03:43:51,084 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732765431084"}]},"ts":"9223372036854775807"} 2024-11-28T03:43:51,085 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T03:43:51,085 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=194, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T03:43:51,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=194, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-28T03:43:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=194 2024-11-28T03:43:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T03:43:51,147 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T03:43:51,156 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 249) - Thread LEAK? -, OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=435 (was 493), ProcessCount=11 (was 11), AvailableMemoryMB=5926 (was 6044) 2024-11-28T03:43:51,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T03:43:51,157 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:43:51,157 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:43:51,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:51,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:51,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:43:51,157 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T03:43:51,157 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=840105357, stopped=false 2024-11-28T03:43:51,157 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3a546af21488,38327,1732765248628 2024-11-28T03:43:51,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T03:43:51,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T03:43:51,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:43:51,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:43:51,160 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T03:43:51,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T03:43:51,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T03:43:51,160 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:43:51,161 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:43:51,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:51,161 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3a546af21488,34671,1732765249447' ***** 2024-11-28T03:43:51,161 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T03:43:51,161 INFO [RS:0;3a546af21488:34671 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T03:43:51,162 INFO [RS:0;3a546af21488:34671 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T03:43:51,162 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T03:43:51,162 INFO [RS:0;3a546af21488:34671 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T03:43:51,162 INFO [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(959): stopping server 3a546af21488,34671,1732765249447 2024-11-28T03:43:51,162 INFO [RS:0;3a546af21488:34671 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T03:43:51,162 INFO [RS:0;3a546af21488:34671 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3a546af21488:34671. 2024-11-28T03:43:51,162 DEBUG [RS:0;3a546af21488:34671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:43:51,162 DEBUG [RS:0;3a546af21488:34671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:51,163 INFO [RS:0;3a546af21488:34671 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T03:43:51,163 INFO [RS:0;3a546af21488:34671 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T03:43:51,163 INFO [RS:0;3a546af21488:34671 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T03:43:51,163 INFO [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T03:43:51,164 INFO [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T03:43:51,164 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-28T03:43:51,164 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T03:43:51,164 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T03:43:51,164 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T03:43:51,164 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T03:43:51,164 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T03:43:51,164 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T03:43:51,164 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-28T03:43:51,188 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/info/3dd54c488745459b9a7f2b76af1bede6 is 95, key is TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c./info:/1732765289200/DeleteFamily/seqid=0 2024-11-28T03:43:51,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742539_1715 (size=6514) 2024-11-28T03:43:51,193 INFO [regionserver/3a546af21488:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T03:43:51,364 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T03:43:51,398 INFO [regionserver/3a546af21488:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-28T03:43:51,398 INFO [regionserver/3a546af21488:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-28T03:43:51,564 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T03:43:51,592 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/info/3dd54c488745459b9a7f2b76af1bede6 2024-11-28T03:43:51,610 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/ns/7057e52a89a045c1b2a4498eefc5c29a is 93, key is TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c./ns:/1732765289136/DeleteFamily/seqid=0 2024-11-28T03:43:51,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742540_1716 (size=5906) 2024-11-28T03:43:51,765 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T03:43:51,965 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T03:43:52,013 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/ns/7057e52a89a045c1b2a4498eefc5c29a 2024-11-28T03:43:52,031 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/rep_barrier/1e7513f7f736451cb3882fb5cadad9be is 102, key is TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c./rep_barrier:/1732765289136/DeleteFamily/seqid=0 2024-11-28T03:43:52,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742541_1717 (size=6025) 2024-11-28T03:43:52,034 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/rep_barrier/1e7513f7f736451cb3882fb5cadad9be 2024-11-28T03:43:52,051 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/table/86c512c5ebe54eeebe721d00b1bacabf is 96, key is TestAcidGuarantees,,1732765253923.b7a40554a92e84435d4dc19dc444ec8c./table:/1732765289136/DeleteFamily/seqid=0 2024-11-28T03:43:52,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742542_1718 (size=5936) 2024-11-28T03:43:52,165 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T03:43:52,365 DEBUG [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T03:43:52,399 INFO [regionserver/3a546af21488:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-28T03:43:52,455 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/table/86c512c5ebe54eeebe721d00b1bacabf 2024-11-28T03:43:52,458 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/info/3dd54c488745459b9a7f2b76af1bede6 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/info/3dd54c488745459b9a7f2b76af1bede6 2024-11-28T03:43:52,461 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/info/3dd54c488745459b9a7f2b76af1bede6, entries=12, sequenceid=90, filesize=6.4 K 2024-11-28T03:43:52,462 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/ns/7057e52a89a045c1b2a4498eefc5c29a as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/ns/7057e52a89a045c1b2a4498eefc5c29a 2024-11-28T03:43:52,464 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/ns/7057e52a89a045c1b2a4498eefc5c29a, entries=8, sequenceid=90, filesize=5.8 K 2024-11-28T03:43:52,465 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/rep_barrier/1e7513f7f736451cb3882fb5cadad9be as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/rep_barrier/1e7513f7f736451cb3882fb5cadad9be 2024-11-28T03:43:52,467 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/rep_barrier/1e7513f7f736451cb3882fb5cadad9be, entries=6, sequenceid=90, filesize=5.9 K 2024-11-28T03:43:52,468 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/.tmp/table/86c512c5ebe54eeebe721d00b1bacabf as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/table/86c512c5ebe54eeebe721d00b1bacabf 2024-11-28T03:43:52,470 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/table/86c512c5ebe54eeebe721d00b1bacabf, entries=7, sequenceid=90, filesize=5.8 K 2024-11-28T03:43:52,471 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1307ms, sequenceid=90, compaction requested=false 2024-11-28T03:43:52,475 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-28T03:43:52,476 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T03:43:52,476 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T03:43:52,476 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732765431164Running coprocessor pre-close hooks at 1732765431164Disabling compacts and flushes for region at 1732765431164Disabling writes for close at 1732765431164Obtaining lock to block concurrent updates at 1732765431164Preparing flush snapshotting stores in 1588230740 at 1732765431164Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732765431165 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732765431166 (+1 ms)Flushing 1588230740/info: creating writer at 1732765431166Flushing 1588230740/info: appending metadata at 1732765431187 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732765431187Flushing 1588230740/ns: creating writer at 1732765431595 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732765431609 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732765431610 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1732765432016 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732765432031 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732765432031Flushing 1588230740/table: creating writer at 1732765432037 (+6 ms)Flushing 1588230740/table: appending metadata at 1732765432051 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732765432051Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bf7821e: reopening flushed file at 1732765432458 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ec3519c: reopening flushed file at 1732765432461 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@130be2e3: reopening flushed file at 1732765432464 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7dc1d26: reopening flushed file at 1732765432468 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1307ms, sequenceid=90, compaction requested=false at 1732765432471 (+3 ms)Writing region close event to WAL at 1732765432473 (+2 ms)Running coprocessor post-close hooks at 1732765432475 (+2 ms)Closed at 1732765432476 (+1 ms) 2024-11-28T03:43:52,476 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T03:43:52,566 INFO [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(976): stopping server 3a546af21488,34671,1732765249447; all regions closed. 2024-11-28T03:43:52,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741834_1010 (size=25344) 2024-11-28T03:43:52,572 DEBUG [RS:0;3a546af21488:34671 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/oldWALs 2024-11-28T03:43:52,572 INFO [RS:0;3a546af21488:34671 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3a546af21488%2C34671%2C1732765249447.meta:.meta(num 1732765253476) 2024-11-28T03:43:52,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741832_1008 (size=19725184) 2024-11-28T03:43:52,575 DEBUG [RS:0;3a546af21488:34671 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/oldWALs 2024-11-28T03:43:52,575 INFO [RS:0;3a546af21488:34671 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3a546af21488%2C34671%2C1732765249447:(num 1732765252557) 2024-11-28T03:43:52,575 DEBUG [RS:0;3a546af21488:34671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:43:52,575 INFO [RS:0;3a546af21488:34671 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T03:43:52,575 INFO [RS:0;3a546af21488:34671 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T03:43:52,576 INFO [RS:0;3a546af21488:34671 {}] hbase.ChoreService(370): Chore service for: regionserver/3a546af21488:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-28T03:43:52,576 INFO [RS:0;3a546af21488:34671 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T03:43:52,576 INFO [regionserver/3a546af21488:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T03:43:52,576 INFO [RS:0;3a546af21488:34671 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34671 2024-11-28T03:43:52,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3a546af21488,34671,1732765249447 2024-11-28T03:43:52,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T03:43:52,581 INFO [RS:0;3a546af21488:34671 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T03:43:52,581 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3a546af21488,34671,1732765249447] 2024-11-28T03:43:52,583 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3a546af21488,34671,1732765249447 already deleted, retry=false 2024-11-28T03:43:52,583 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3a546af21488,34671,1732765249447 expired; onlineServers=0 2024-11-28T03:43:52,583 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3a546af21488,38327,1732765248628' ***** 2024-11-28T03:43:52,583 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T03:43:52,584 INFO [M:0;3a546af21488:38327 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T03:43:52,584 INFO [M:0;3a546af21488:38327 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T03:43:52,584 DEBUG [M:0;3a546af21488:38327 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T03:43:52,584 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T03:43:52,584 DEBUG [M:0;3a546af21488:38327 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T03:43:52,584 DEBUG [master/3a546af21488:0:becomeActiveMaster-HFileCleaner.large.0-1732765252224 {}] cleaner.HFileCleaner(306): Exit Thread[master/3a546af21488:0:becomeActiveMaster-HFileCleaner.large.0-1732765252224,5,FailOnTimeoutGroup] 2024-11-28T03:43:52,584 DEBUG [master/3a546af21488:0:becomeActiveMaster-HFileCleaner.small.0-1732765252224 {}] cleaner.HFileCleaner(306): Exit Thread[master/3a546af21488:0:becomeActiveMaster-HFileCleaner.small.0-1732765252224,5,FailOnTimeoutGroup] 2024-11-28T03:43:52,584 INFO [M:0;3a546af21488:38327 {}] hbase.ChoreService(370): Chore service for: master/3a546af21488:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T03:43:52,584 INFO [M:0;3a546af21488:38327 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T03:43:52,584 DEBUG [M:0;3a546af21488:38327 {}] master.HMaster(1795): Stopping service threads 2024-11-28T03:43:52,585 INFO [M:0;3a546af21488:38327 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T03:43:52,585 INFO [M:0;3a546af21488:38327 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T03:43:52,585 ERROR [M:0;3a546af21488:38327 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-28T03:43:52,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T03:43:52,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:43:52,586 INFO [M:0;3a546af21488:38327 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T03:43:52,586 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T03:43:52,586 DEBUG [M:0;3a546af21488:38327 {}] zookeeper.ZKUtil(347): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T03:43:52,586 WARN [M:0;3a546af21488:38327 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T03:43:52,587 INFO [M:0;3a546af21488:38327 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/.lastflushedseqids 2024-11-28T03:43:52,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742543_1719 (size=72) 2024-11-28T03:43:52,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T03:43:52,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34671-0x1003c3721c70001, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T03:43:52,684 INFO [RS:0;3a546af21488:34671 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T03:43:52,684 INFO [RS:0;3a546af21488:34671 {}] regionserver.HRegionServer(1031): Exiting; stopping=3a546af21488,34671,1732765249447; zookeeper connection closed. 2024-11-28T03:43:52,685 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@48db4da9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@48db4da9 2024-11-28T03:43:52,685 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-28T03:43:52,997 INFO [M:0;3a546af21488:38327 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T03:43:52,997 INFO [M:0;3a546af21488:38327 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T03:43:52,997 DEBUG [M:0;3a546af21488:38327 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T03:43:52,997 INFO [M:0;3a546af21488:38327 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:43:52,997 DEBUG [M:0;3a546af21488:38327 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:43:52,997 DEBUG [M:0;3a546af21488:38327 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T03:43:52,997 DEBUG [M:0;3a546af21488:38327 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:43:52,998 INFO [M:0;3a546af21488:38327 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=829.52 KB heapSize=1019.26 KB 2024-11-28T03:43:53,014 DEBUG [M:0;3a546af21488:38327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a399d84a9db44aaebddd7266e499a840 is 82, key is hbase:meta,,1/info:regioninfo/1732765253565/Put/seqid=0 2024-11-28T03:43:53,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742544_1720 (size=5672) 2024-11-28T03:43:53,417 INFO [M:0;3a546af21488:38327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2336 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a399d84a9db44aaebddd7266e499a840 2024-11-28T03:43:53,438 DEBUG [M:0;3a546af21488:38327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed93d42ef8db49e2bd52de985ff8f2b0 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\xA3/proc:d/1732765407152/Put/seqid=0 2024-11-28T03:43:53,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742545_1721 (size=49546) 2024-11-28T03:43:53,842 INFO [M:0;3a546af21488:38327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=828.96 KB at sequenceid=2336 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed93d42ef8db49e2bd52de985ff8f2b0 2024-11-28T03:43:53,845 INFO [M:0;3a546af21488:38327 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ed93d42ef8db49e2bd52de985ff8f2b0 2024-11-28T03:43:53,863 DEBUG [M:0;3a546af21488:38327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/915f5df56b004dc4a2615ed0e431fbd4 is 69, key is 3a546af21488,34671,1732765249447/rs:state/1732765252329/Put/seqid=0 2024-11-28T03:43:53,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073742546_1722 (size=5156) 2024-11-28T03:43:54,266 INFO [M:0;3a546af21488:38327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2336 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/915f5df56b004dc4a2615ed0e431fbd4 2024-11-28T03:43:54,270 DEBUG [M:0;3a546af21488:38327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a399d84a9db44aaebddd7266e499a840 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a399d84a9db44aaebddd7266e499a840 2024-11-28T03:43:54,272 INFO [M:0;3a546af21488:38327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a399d84a9db44aaebddd7266e499a840, entries=8, sequenceid=2336, filesize=5.5 K 2024-11-28T03:43:54,273 DEBUG [M:0;3a546af21488:38327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed93d42ef8db49e2bd52de985ff8f2b0 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ed93d42ef8db49e2bd52de985ff8f2b0 2024-11-28T03:43:54,275 INFO [M:0;3a546af21488:38327 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ed93d42ef8db49e2bd52de985ff8f2b0 2024-11-28T03:43:54,275 INFO [M:0;3a546af21488:38327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ed93d42ef8db49e2bd52de985ff8f2b0, entries=194, sequenceid=2336, filesize=48.4 K 2024-11-28T03:43:54,275 DEBUG [M:0;3a546af21488:38327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/915f5df56b004dc4a2615ed0e431fbd4 as hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/915f5df56b004dc4a2615ed0e431fbd4 2024-11-28T03:43:54,278 INFO [M:0;3a546af21488:38327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38161/user/jenkins/test-data/c6891f19-3677-03b8-85aa-a6d2f47c8d72/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/915f5df56b004dc4a2615ed0e431fbd4, entries=1, sequenceid=2336, filesize=5.0 K 2024-11-28T03:43:54,278 INFO [M:0;3a546af21488:38327 {}] regionserver.HRegion(3140): Finished flush of dataSize ~829.52 KB/849428, heapSize ~1018.96 KB/1043416, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1281ms, sequenceid=2336, compaction requested=false 2024-11-28T03:43:54,280 INFO [M:0;3a546af21488:38327 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:43:54,280 DEBUG [M:0;3a546af21488:38327 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732765432997Disabling compacts and flushes for region at 1732765432997Disabling writes for close at 1732765432997Obtaining lock to block concurrent updates at 1732765432998 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732765432998Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=849428, getHeapSize=1043656, getOffHeapSize=0, getCellsCount=2555 at 1732765432998Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732765432998Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732765432999 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732765433014 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732765433014Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732765433421 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732765433438 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732765433438Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732765433845 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732765433862 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732765433862Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15318de5: reopening flushed file at 1732765434269 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a27be2b: reopening flushed file at 1732765434272 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@727d8961: reopening flushed file at 1732765434275 (+3 ms)Finished flush of dataSize ~829.52 KB/849428, heapSize ~1018.96 KB/1043416, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1281ms, sequenceid=2336, compaction requested=false at 1732765434278 (+3 ms)Writing region close event to WAL at 1732765434280 (+2 ms)Closed at 1732765434280 2024-11-28T03:43:54,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35257 is added to blk_1073741830_1006 (size=1002563) 2024-11-28T03:43:54,282 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T03:43:54,283 INFO [M:0;3a546af21488:38327 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T03:43:54,283 INFO [M:0;3a546af21488:38327 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38327 2024-11-28T03:43:54,283 INFO [M:0;3a546af21488:38327 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T03:43:54,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T03:43:54,385 INFO [M:0;3a546af21488:38327 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T03:43:54,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38327-0x1003c3721c70000, quorum=127.0.0.1:50439, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T03:43:54,390 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58e0e813{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T03:43:54,393 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@716e8981{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T03:43:54,393 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T03:43:54,393 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@50fd64ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T03:43:54,393 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d058786{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/hadoop.log.dir/,STOPPED} 2024-11-28T03:43:54,396 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-28T03:43:54,396 WARN [BP-1741671718-172.17.0.2-1732765245159 heartbeating to localhost/127.0.0.1:38161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-28T03:43:54,396 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-28T03:43:54,397 WARN [BP-1741671718-172.17.0.2-1732765245159 heartbeating to localhost/127.0.0.1:38161 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1741671718-172.17.0.2-1732765245159 (Datanode Uuid 1af0cbc8-7da9-4027-a904-1e366931edcb) service to localhost/127.0.0.1:38161 2024-11-28T03:43:54,399 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/cluster_bc0417c7-3664-f9fe-1add-573adc96de9e/data/data1/current/BP-1741671718-172.17.0.2-1732765245159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T03:43:54,399 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/cluster_bc0417c7-3664-f9fe-1add-573adc96de9e/data/data2/current/BP-1741671718-172.17.0.2-1732765245159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T03:43:54,400 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-28T03:43:54,407 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a5ffc6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T03:43:54,407 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f30a77e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T03:43:54,408 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T03:43:54,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40029c9a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T03:43:54,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f9fd429{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a678640-1d2a-56ec-1230-977a421b6b64/hadoop.log.dir/,STOPPED} 2024-11-28T03:43:54,425 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T03:43:54,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down